From a030be3d7e9f357bc55e1d7a06cebc1676e41d91 Mon Sep 17 00:00:00 2001 From: Gary Russell Date: Thu, 25 Feb 2016 16:24:23 -0500 Subject: [PATCH] Use spring-kafka in s-i-kafka Update to s-i 1.0.0.M1 Polishing Fix Readme --- .../xml/KafkaConsumerContextParser.java | 156 ----- .../xml/KafkaInboundChannelAdapterParser.java | 53 -- ...afkaMessageDrivenChannelAdapterParser.java | 57 +- .../config/xml/KafkaNamespaceHandler.java | 8 +- .../KafkaOutboundChannelAdapterParser.java | 11 +- .../xml/KafkaProducerContextParser.java | 172 ----- .../config/xml/ZookeeperConnectParser.java | 47 -- .../kafka/core/AbstractConfiguration.java | 187 ----- .../integration/kafka/core/BrokerAddress.java | 100 --- .../core/BrokerAddressListConfiguration.java | 41 -- .../integration/kafka/core/Configuration.java | 101 --- .../integration/kafka/core/Connection.java | 92 --- .../kafka/core/ConnectionFactory.java | 73 -- .../kafka/core/ConsumerException.java | 41 -- .../kafka/core/DefaultConnection.java | 324 --------- .../kafka/core/DefaultConnectionFactory.java | 276 -------- .../integration/kafka/core/FetchRequest.java | 62 -- .../kafka/core/KafkaConsumerDefaults.java | 70 -- .../integration/kafka/core/KafkaMessage.java | 53 -- .../kafka/core/KafkaMessageBatch.java | 64 -- .../kafka/core/KafkaMessageMetadata.java | 65 -- .../kafka/core/KafkaOperations.java | 31 - .../integration/kafka/core/KafkaTemplate.java | 65 -- .../integration/kafka/core/MetadataCache.java | 111 --- .../integration/kafka/core/Partition.java | 82 --- .../core/PartitionNotFoundException.java | 30 - .../integration/kafka/core/Result.java | 64 -- .../integration/kafka/core/ResultBuilder.java | 74 -- .../kafka/core/TopicNotFoundException.java | 29 - .../kafka/core/ZookeeperConfiguration.java | 106 --- .../kafka/core/ZookeeperConnectDefaults.java | 32 - .../integration/kafka/core/package-info.java | 4 - .../KafkaHighLevelConsumerMessageSource.java | 49 -- .../KafkaMessageDrivenChannelAdapter.java | 135 ++-- ...tDecodingAcknowledgingMessageListener.java | 62 -- .../AbstractDecodingMessageListener.java | 62 -- .../kafka/listener/AbstractOffsetManager.java | 176 ----- .../AcknowledgingMessageListener.java | 38 - .../kafka/listener/Acknowledgment.java | 37 - .../ConcurrentMessageListenerDispatcher.java | 167 ----- .../kafka/listener/DefaultAcknowledgment.java | 52 -- .../kafka/listener/ErrorHandler.java | 31 - .../KafkaMessageListenerContainer.java | 655 ------------------ .../listener/KafkaNativeOffsetManager.java | 231 ------ .../listener/KafkaTopicOffsetManager.java | 506 -------------- .../kafka/listener/LoggingErrorHandler.java | 38 - .../kafka/listener/LongSerializerDecoder.java | 76 -- .../kafka/listener/MessageListener.java | 35 - .../listener/MetadataStoreOffsetManager.java | 99 --- .../kafka/listener/OffsetManager.java | 61 -- .../QueueingMessageListenerInvoker.java | 264 ------- .../listener/WindowingOffsetManager.java | 310 --------- .../kafka/listener/package-info.java | 4 - .../outbound/KafkaProducerMessageHandler.java | 27 +- .../serializer/avro/AvroDatumSupport.java | 41 -- .../AvroReflectDatumBackedKafkaDecoder.java | 39 -- .../AvroReflectDatumBackedKafkaEncoder.java | 39 -- .../kafka/serializer/avro/AvroSerializer.java | 48 -- .../AvroSpecificDatumBackedKafkaDecoder.java | 24 - .../AvroSpecificDatumBackedKafkaEncoder.java | 24 - .../serializer/common/StringDecoder.java | 50 -- .../serializer/common/StringEncoder.java | 47 -- .../support/ConsumerConfigFactoryBean.java | 85 --- .../kafka/support/ConsumerConfiguration.java | 280 -------- .../support/ConsumerConnectionProvider.java | 37 - .../kafka/support/ConsumerMetadata.java | 203 ------ .../kafka/support/DefaultPartitioner.java | 39 -- .../support/DefaultProducerListener.java | 36 - .../kafka/support/KafkaConsumerContext.java | 92 --- .../kafka/support/KafkaHeaders.java | 40 -- .../kafka/support/KafkaProducerContext.java | 215 ------ .../support/LoggingProducerListener.java | 83 --- .../kafka/support/MessageLeftOverTracker.java | 46 -- .../kafka/support/ProducerConfiguration.java | 201 ------ .../kafka/support/ProducerFactoryBean.java | 79 --- .../kafka/support/ProducerListener.java | 55 -- .../ProducerListenerInvokingCallback.java | 61 -- .../kafka/support/ProducerMetadata.java | 173 ----- .../support/TopicFilterConfiguration.java | 58 -- .../kafka/support/ZookeeperConnect.java | 72 -- .../kafka/support/package-info.java | 4 - .../kafka/util/EncoderAdaptingSerializer.java | 57 -- .../integration/kafka/util/LoggingUtils.java | 45 -- .../integration/kafka/util/MessageUtils.java | 43 -- .../integration/kafka/util/TopicUtils.java | 160 ----- .../config/spring-integration-kafka-2.0.xsd | 639 +---------------- ...afkaConsumerContextParserTests-context.xml | 54 -- .../xml/KafkaConsumerContextParserTests.java | 72 -- ...KafkaInboundAdapterParserTests-context.xml | 40 -- .../xml/KafkaInboundAdapterParserTests.java | 58 -- ...rivenChannelAdapterParserTests-context.xml | 138 +--- ...essageDrivenChannelAdapterParserTests.java | 151 +--- ...ultiConsumerContextParserTests-context.xml | 69 -- .../KafkaMultiConsumerContextParserTests.java | 86 --- ...afkaOutboundAdapterParserTests-context.xml | 51 +- .../xml/KafkaOutboundAdapterParserTests.java | 36 +- ...afkaProducerContextParserTests-context.xml | 64 -- .../xml/KafkaProducerContextParserTests.java | 143 ---- .../ZookeeperConnectParserTests-context.xml | 14 - .../xml/ZookeeperConnectParserTests.java | 65 -- .../xml/kafkaInboundAdapterCommon-context.xml | 18 - .../kafka/core/MetadataCacheTests.java | 262 ------- .../inbound/MessageDrivenAdapterTests.java | 75 ++ .../kafka/listener/AbstractBrokerTests.java | 220 ------ ...AbstractMessageListenerContainerTests.java | 238 ------- .../ChannelAdapterShutdownTests-context.xml | 47 -- .../listener/ChannelAdapterShutdownTests.java | 92 --- ...apterWithXmlConfigurationTests-context.xml | 39 -- ...annelAdapterWithXmlConfigurationTests.java | 68 -- .../ChannelSendingMessageListener.java | 61 -- .../DefaultConnectionFactoryTests.java | 102 --- .../listener/DefaultConnectionTests.java | 133 ---- ...KafkaMessageDrivenChannelAdapterTests.java | 246 ------- ...nelAdapterWithKafkaOffsetManagerTests.java | 139 ---- ...nChannelAdapterWithSpecialOffsetTests.java | 172 ----- ...venChannelAdapterWithWrongOffsetTests.java | 327 --------- .../kafka/listener/MultiBroker20Tests.java | 60 -- .../kafka/listener/MultiBroker2Tests.java | 103 --- .../listener/MultiBroker5ReplicatedTests.java | 102 --- ...MultiBroker5ReplicatedWithBounceTests.java | 119 ---- .../kafka/listener/MultiBroker5Tests.java | 102 --- .../listener/NewBrokerRecoveryTests.java | 136 ---- .../listener/SingleBrokerExternalTests.java | 131 ---- .../listener/SingleBrokerRecoveryTests.java | 118 ---- .../kafka/listener/SingleBrokerTests.java | 98 --- .../SingleBrokerWithCompressionTests.java | 98 --- .../SingleBrokerWithManualAckTests.java | 149 ---- .../SingleBrokerWithPartitionSubsetTests.java | 103 --- .../kafka/listener/TestPartitioner.java | 50 -- .../kafka/listener/WrongTopicNameTests.java | 90 --- .../listener/ZookeeperConfigurationTests.java | 110 --- .../offset/AbstractOffsetManagerTests.java | 302 -------- .../integration/kafka/offset/CodecTests.java | 52 -- .../offset/KafkaNativeOffsetManagerTests.java | 48 -- .../offset/KafkaTopicOffsetManagerTests.java | 44 -- .../MetadataStoreOffsetManagerTests.java | 53 -- .../offset/WindowingOffsetManagerTests.java | 54 -- .../kafka/outbound/OutboundTests.java | 465 ------------- .../kafka/outbound/ProducerListenerTests.java | 105 --- .../kafka/outbound/ProducerTest.java | 60 -- .../OffsetManagerPerformanceTests.java | 154 ---- .../integration/kafka/rule/KafkaEmbedded.java | 311 --------- .../integration/kafka/rule/KafkaRule.java | 46 -- .../integration/kafka/rule/KafkaRunning.java | 129 ---- ...ReflectDatumBackedKafkaSerializerTest.java | 58 -- ...pecificDatumBackedKafkaSerializerTest.java | 29 - .../support/ConsumerConfigurationTests.java | 538 -------------- .../KafkaConsumerContextIntegrationTests.java | 142 ---- .../support/KafkaConsumerContextTest.java | 86 --- .../support/KafkaProducerContextTests.java | 59 -- .../support/ProducerConfigurationTests.java | 345 --------- .../support/ProducerFactoryBeanTests.java | 43 -- .../integration/kafka/test/utils/User.java | 93 --- .../src/test/resources/log4j.properties | 10 - 154 files changed, 205 insertions(+), 16781 deletions(-) delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaConsumerContextParser.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaInboundChannelAdapterParser.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaProducerContextParser.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/ZookeeperConnectParser.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/AbstractConfiguration.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/BrokerAddress.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/BrokerAddressListConfiguration.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/Configuration.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/Connection.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ConnectionFactory.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ConsumerException.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/DefaultConnection.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/DefaultConnectionFactory.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/FetchRequest.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaConsumerDefaults.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaMessage.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaMessageBatch.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaMessageMetadata.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaOperations.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaTemplate.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/MetadataCache.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/Partition.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/PartitionNotFoundException.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/Result.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ResultBuilder.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/TopicNotFoundException.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ZookeeperConfiguration.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ZookeeperConnectDefaults.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/package-info.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/inbound/KafkaHighLevelConsumerMessageSource.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/AbstractDecodingAcknowledgingMessageListener.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/AbstractDecodingMessageListener.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/AbstractOffsetManager.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/AcknowledgingMessageListener.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/Acknowledgment.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/ConcurrentMessageListenerDispatcher.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/DefaultAcknowledgment.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/ErrorHandler.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/KafkaMessageListenerContainer.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/KafkaNativeOffsetManager.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/KafkaTopicOffsetManager.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/LoggingErrorHandler.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/LongSerializerDecoder.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/MessageListener.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/MetadataStoreOffsetManager.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/OffsetManager.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/QueueingMessageListenerInvoker.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/WindowingOffsetManager.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/package-info.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/avro/AvroDatumSupport.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/avro/AvroReflectDatumBackedKafkaDecoder.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/avro/AvroReflectDatumBackedKafkaEncoder.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/avro/AvroSerializer.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/avro/AvroSpecificDatumBackedKafkaDecoder.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/avro/AvroSpecificDatumBackedKafkaEncoder.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/common/StringDecoder.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/common/StringEncoder.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ConsumerConfigFactoryBean.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ConsumerConfiguration.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ConsumerConnectionProvider.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ConsumerMetadata.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/DefaultPartitioner.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/DefaultProducerListener.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/KafkaConsumerContext.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/KafkaHeaders.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/KafkaProducerContext.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/LoggingProducerListener.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/MessageLeftOverTracker.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ProducerConfiguration.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ProducerFactoryBean.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ProducerListener.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ProducerListenerInvokingCallback.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ProducerMetadata.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/TopicFilterConfiguration.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ZookeeperConnect.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/package-info.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/util/EncoderAdaptingSerializer.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/util/LoggingUtils.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/util/MessageUtils.java delete mode 100644 spring-integration-kafka/src/main/java/org/springframework/integration/kafka/util/TopicUtils.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaConsumerContextParserTests-context.xml delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaConsumerContextParserTests.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaInboundAdapterParserTests-context.xml delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaInboundAdapterParserTests.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaMultiConsumerContextParserTests-context.xml delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaMultiConsumerContextParserTests.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaProducerContextParserTests-context.xml delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaProducerContextParserTests.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/ZookeeperConnectParserTests-context.xml delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/ZookeeperConnectParserTests.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/kafkaInboundAdapterCommon-context.xml delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/core/MetadataCacheTests.java create mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/inbound/MessageDrivenAdapterTests.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/AbstractBrokerTests.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/AbstractMessageListenerContainerTests.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/ChannelAdapterShutdownTests-context.xml delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/ChannelAdapterShutdownTests.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/ChannelAdapterWithXmlConfigurationTests-context.xml delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/ChannelAdapterWithXmlConfigurationTests.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/ChannelSendingMessageListener.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/DefaultConnectionFactoryTests.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/DefaultConnectionTests.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/KafkaMessageDrivenChannelAdapterTests.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/KafkaMessageDrivenChannelAdapterWithKafkaOffsetManagerTests.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/KafkaMessageDrivenChannelAdapterWithSpecialOffsetTests.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/KafkaMessageDrivenChannelAdapterWithWrongOffsetTests.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker20Tests.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker2Tests.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker5ReplicatedTests.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker5ReplicatedWithBounceTests.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker5Tests.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/NewBrokerRecoveryTests.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/SingleBrokerExternalTests.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/SingleBrokerRecoveryTests.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/SingleBrokerTests.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/SingleBrokerWithCompressionTests.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/SingleBrokerWithManualAckTests.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/SingleBrokerWithPartitionSubsetTests.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/TestPartitioner.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/WrongTopicNameTests.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/ZookeeperConfigurationTests.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/offset/AbstractOffsetManagerTests.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/offset/CodecTests.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/offset/KafkaNativeOffsetManagerTests.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/offset/KafkaTopicOffsetManagerTests.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/offset/MetadataStoreOffsetManagerTests.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/offset/WindowingOffsetManagerTests.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/outbound/OutboundTests.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/outbound/ProducerListenerTests.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/outbound/ProducerTest.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/performance/OffsetManagerPerformanceTests.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/rule/KafkaEmbedded.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/rule/KafkaRule.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/rule/KafkaRunning.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/serializer/AvroReflectDatumBackedKafkaSerializerTest.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/serializer/AvroSpecificDatumBackedKafkaSerializerTest.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/support/ConsumerConfigurationTests.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/support/KafkaConsumerContextIntegrationTests.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/support/KafkaConsumerContextTest.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/support/KafkaProducerContextTests.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/support/ProducerConfigurationTests.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/support/ProducerFactoryBeanTests.java delete mode 100644 spring-integration-kafka/src/test/java/org/springframework/integration/kafka/test/utils/User.java delete mode 100644 spring-integration-kafka/src/test/resources/log4j.properties diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaConsumerContextParser.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaConsumerContextParser.java deleted file mode 100644 index 64b1fe8b37..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaConsumerContextParser.java +++ /dev/null @@ -1,156 +0,0 @@ -/* - * Copyright 2002-2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.config.xml; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.w3c.dom.Element; - -import org.springframework.beans.BeanMetadataElement; -import org.springframework.beans.factory.config.BeanDefinition; -import org.springframework.beans.factory.support.AbstractBeanDefinition; -import org.springframework.beans.factory.support.BeanDefinitionBuilder; -import org.springframework.beans.factory.support.ManagedMap; -import org.springframework.beans.factory.xml.AbstractSingleBeanDefinitionParser; -import org.springframework.beans.factory.xml.ParserContext; -import org.springframework.integration.config.xml.IntegrationNamespaceUtils; -import org.springframework.util.StringUtils; -import org.springframework.util.xml.DomUtils; - -/** - * @author Soby Chacko - * @author Rajasekar Elango - * @author Artem Bilan - * @author Ilayaperumal Gopinathan - * @author Gary Russell - * @since 0.5 - */ -@Deprecated -@SuppressWarnings("deprecation") -public class KafkaConsumerContextParser extends AbstractSingleBeanDefinitionParser { - - @Override - protected Class getBeanClass(final Element element) { - return org.springframework.integration.kafka.support.KafkaConsumerContext.class; - } - - @Override - protected void doParse(final Element element, final ParserContext parserContext, final BeanDefinitionBuilder builder) { - super.doParse(element, parserContext, builder); - - final Element consumerConfigurations = DomUtils.getChildElementByTagName(element, "consumer-configurations"); - parseConsumerConfigurations(consumerConfigurations, parserContext, builder, element); - } - - private void parseConsumerConfigurations(final Element consumerConfigurations, final ParserContext parserContext, - final BeanDefinitionBuilder builder, final Element parentElem) { - Map consumerConfigurationsMap = new ManagedMap(); - for (final Element consumerConfiguration : DomUtils.getChildElementsByTagName(consumerConfigurations, "consumer-configuration")) { - final BeanDefinitionBuilder consumerConfigurationBuilder = - BeanDefinitionBuilder.genericBeanDefinition(org.springframework.integration.kafka.support.ConsumerConfiguration.class); - final BeanDefinitionBuilder consumerMetadataBuilder = - BeanDefinitionBuilder.genericBeanDefinition(org.springframework.integration.kafka.support.ConsumerMetadata.class); - - IntegrationNamespaceUtils.setValueIfAttributeDefined(consumerMetadataBuilder, consumerConfiguration, - "group-id"); - - IntegrationNamespaceUtils.setReferenceIfAttributeDefined(consumerMetadataBuilder, consumerConfiguration, - "value-decoder"); - IntegrationNamespaceUtils.setReferenceIfAttributeDefined(consumerMetadataBuilder, consumerConfiguration, - "key-decoder"); - IntegrationNamespaceUtils.setValueIfAttributeDefined(consumerConfigurationBuilder, consumerConfiguration, - "max-messages"); - IntegrationNamespaceUtils.setReferenceIfAttributeDefined(consumerConfigurationBuilder, consumerConfiguration, - "executor"); - IntegrationNamespaceUtils.setValueIfAttributeDefined(consumerMetadataBuilder, parentElem, - "consumer-timeout"); - - final Map topicStreamsMap = new HashMap(); - - final List topicConfigurations = DomUtils.getChildElementsByTagName(consumerConfiguration, "topic"); - - if (topicConfigurations != null) { - for (final Element topicConfiguration : topicConfigurations) { - final String topic = topicConfiguration.getAttribute("id"); - final String streams = topicConfiguration.getAttribute("streams"); - topicStreamsMap.put(topic, streams); - } - consumerMetadataBuilder.addPropertyValue("topicStreamMap", topicStreamsMap); - } - - final Element topicFilter = DomUtils.getChildElementByTagName(consumerConfiguration, "topic-filter"); - - if (topicFilter != null) { - BeanDefinition topicFilterConfigurationBeanDefinition = - BeanDefinitionBuilder.genericBeanDefinition(org.springframework.integration.kafka.support.TopicFilterConfiguration.class) - .addConstructorArgValue(topicFilter.getAttribute("pattern")) - .addConstructorArgValue(topicFilter.getAttribute("streams")) - .addConstructorArgValue(topicFilter.getAttribute("exclude")) - .getBeanDefinition(); - consumerMetadataBuilder.addPropertyValue("topicFilterConfiguration", - topicFilterConfigurationBeanDefinition); - } - - final AbstractBeanDefinition consumerMetadataBeanDefintiion = consumerMetadataBuilder.getBeanDefinition(); - - final String zookeeperConnectBean = parentElem.getAttribute("zookeeper-connect"); - IntegrationNamespaceUtils.setReferenceIfAttributeDefined(builder, parentElem, zookeeperConnectBean); - - final String consumerPropertiesBean = parentElem.getAttribute("consumer-properties"); - - final BeanDefinitionBuilder consumerConfigFactoryBuilder = - BeanDefinitionBuilder.genericBeanDefinition(org.springframework.integration.kafka.support.ConsumerConfigFactoryBean.class); - consumerConfigFactoryBuilder.addConstructorArgValue(consumerMetadataBeanDefintiion); - - if (StringUtils.hasText(zookeeperConnectBean)) { - consumerConfigFactoryBuilder.addConstructorArgReference(zookeeperConnectBean); - } - - if (StringUtils.hasText(consumerPropertiesBean)) { - consumerConfigFactoryBuilder.addConstructorArgReference(consumerPropertiesBean); - } - - AbstractBeanDefinition consumerConfigFactoryBuilderBeanDefinition = - consumerConfigFactoryBuilder.getBeanDefinition(); - - BeanDefinitionBuilder consumerConnectionProviderBuilder = - BeanDefinitionBuilder.genericBeanDefinition(org.springframework.integration.kafka.support.ConsumerConnectionProvider.class); - consumerConnectionProviderBuilder.addConstructorArgValue(consumerConfigFactoryBuilderBeanDefinition); - - AbstractBeanDefinition consumerConnectionProviderBuilderBeanDefinition = - consumerConnectionProviderBuilder.getBeanDefinition(); - - BeanDefinitionBuilder messageLeftOverBeanDefinitionBuilder = - BeanDefinitionBuilder.genericBeanDefinition(org.springframework.integration.kafka.support.MessageLeftOverTracker.class); - AbstractBeanDefinition messageLeftOverBeanDefinition = - messageLeftOverBeanDefinitionBuilder.getBeanDefinition(); - - consumerConfigurationBuilder.addConstructorArgValue(consumerMetadataBeanDefintiion); - consumerConfigurationBuilder.addConstructorArgValue(consumerConnectionProviderBuilderBeanDefinition); - consumerConfigurationBuilder.addConstructorArgValue(messageLeftOverBeanDefinition); - - AbstractBeanDefinition consumerConfigurationBeanDefinition = - consumerConfigurationBuilder.getBeanDefinition(); - consumerConfigurationsMap.put(consumerConfiguration.getAttribute("group-id"), - consumerConfigurationBeanDefinition); - } - builder.addPropertyValue("consumerConfigurations", consumerConfigurationsMap); - } - -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaInboundChannelAdapterParser.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaInboundChannelAdapterParser.java deleted file mode 100644 index ce338b3131..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaInboundChannelAdapterParser.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.kafka.config.xml; - -import org.w3c.dom.Element; - -import org.springframework.beans.BeanMetadataElement; -import org.springframework.beans.factory.support.BeanDefinitionBuilder; -import org.springframework.beans.factory.xml.ParserContext; -import org.springframework.integration.config.xml.AbstractPollingInboundChannelAdapterParser; -import org.springframework.integration.config.xml.IntegrationNamespaceUtils; -import org.springframework.util.StringUtils; - -/** - * The Kafka Inbound Channel adapter parser - * - * @author Soby Chacko - * @since 0.5 - * - */ -@Deprecated -public class KafkaInboundChannelAdapterParser extends AbstractPollingInboundChannelAdapterParser { - - @Override - @SuppressWarnings("deprecation") - protected BeanMetadataElement parseSource(final Element element, final ParserContext parserContext) { - final BeanDefinitionBuilder highLevelConsumerMessageSourceBuilder = - BeanDefinitionBuilder.genericBeanDefinition(org.springframework.integration.kafka.inbound.KafkaHighLevelConsumerMessageSource.class); - - IntegrationNamespaceUtils.setReferenceIfAttributeDefined(highLevelConsumerMessageSourceBuilder, element, "kafka-decoder"); - - final String kafkaConsumerContext = element.getAttribute("kafka-consumer-context-ref"); - - if (StringUtils.hasText(kafkaConsumerContext)) { - highLevelConsumerMessageSourceBuilder.addConstructorArgReference(kafkaConsumerContext); - } - - return highLevelConsumerMessageSourceBuilder.getBeanDefinition(); - } -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaMessageDrivenChannelAdapterParser.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaMessageDrivenChannelAdapterParser.java index 526d6c097a..bb0029ada0 100644 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaMessageDrivenChannelAdapterParser.java +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaMessageDrivenChannelAdapterParser.java @@ -1,5 +1,5 @@ /* - * Copyright 2015 the original author or authors + * Copyright 2015-2016 the original author or authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -24,11 +24,11 @@ import org.springframework.beans.factory.xml.ParserContext; import org.springframework.integration.config.xml.AbstractChannelAdapterParser; import org.springframework.integration.config.xml.IntegrationNamespaceUtils; import org.springframework.integration.kafka.inbound.KafkaMessageDrivenChannelAdapter; -import org.springframework.integration.kafka.listener.KafkaMessageListenerContainer; import org.springframework.util.StringUtils; /** - * @author Artem Bilan. + * @author Artem Bilan + * @author Gary Russell */ public class KafkaMessageDrivenChannelAdapterParser extends AbstractChannelAdapterParser { @@ -38,65 +38,16 @@ public class KafkaMessageDrivenChannelAdapterParser extends AbstractChannelAdapt BeanDefinitionBuilder.genericBeanDefinition(KafkaMessageDrivenChannelAdapter.class); String container = element.getAttribute("listener-container"); - String connectionFactory = element.getAttribute("connection-factory"); - String topics = element.getAttribute("topics"); - String offsetManager = element.getAttribute("offset-manager"); - String errorHandler = element.getAttribute("error-handler"); - String taskExecutor = element.getAttribute("task-executor"); - String concurrency = element.getAttribute("concurrency"); - String stopTimeout = element.getAttribute("stop-timeout"); - String maxFetch = element.getAttribute("max-fetch"); - String queueSize = element.getAttribute("queue-size"); - - if (StringUtils.hasText(container) && - (StringUtils.hasText(connectionFactory) || StringUtils.hasText(topics) - || StringUtils.hasText(offsetManager) || StringUtils.hasText(errorHandler) - || StringUtils.hasText(taskExecutor) || StringUtils.hasText(concurrency) - || StringUtils.hasText(maxFetch) || StringUtils.hasText(queueSize) - || StringUtils.hasText(stopTimeout))) { - parserContext.getReaderContext().error("The 'listener-container' is mutually exclusive with " + - "'connection-factory', 'topics', 'offset-manager', 'error-handler', 'task-executor', " + - "'concurrency', 'stop-timeout', 'max-fetch' and 'queue-size'.", element); - } - if (StringUtils.hasText(container)) { builder.addConstructorArgReference(container); } else { - if (!StringUtils.hasText(connectionFactory)) { - parserContext.getReaderContext().error("The 'connection-factory' attribute is required when " + - "'listener-container' isn't provided.", element); - } - if (!StringUtils.hasText(topics)) { - parserContext.getReaderContext().error("The 'topics' attribute is required when " + - "'listener-container' isn't provided.", element); - } - - BeanDefinitionBuilder containerBuilder = - BeanDefinitionBuilder.genericBeanDefinition(KafkaMessageListenerContainer.class); - - containerBuilder.addConstructorArgReference(connectionFactory) - .addConstructorArgValue(topics); - IntegrationNamespaceUtils.setReferenceIfAttributeDefined(containerBuilder, element, "offset-manager"); - IntegrationNamespaceUtils.setReferenceIfAttributeDefined(containerBuilder, element, "error-handler"); - IntegrationNamespaceUtils.setReferenceIfAttributeDefined( - containerBuilder, element, "task-executor", "fetchTaskExecutor"); - IntegrationNamespaceUtils.setValueIfAttributeDefined(containerBuilder, element, "concurrency"); - IntegrationNamespaceUtils.setValueIfAttributeDefined(containerBuilder, element, "max-fetch"); - IntegrationNamespaceUtils.setValueIfAttributeDefined(containerBuilder, element, "stop-timeout"); - IntegrationNamespaceUtils.setValueIfAttributeDefined(containerBuilder, element, "queue-size"); - IntegrationNamespaceUtils.setValueIfAttributeDefined(containerBuilder, element, "auto-commit-on-error"); - builder.addConstructorArgValue(containerBuilder.getBeanDefinition()); + parserContext.getReaderContext().error("The 'listener-container' attribute is required.", element); } builder.addPropertyReference("outputChannel", channelName); IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element, "send-timeout"); IntegrationNamespaceUtils.setReferenceIfAttributeDefined(builder, element, "error-channel"); - IntegrationNamespaceUtils.setReferenceIfAttributeDefined(builder, element, "key-decoder"); - IntegrationNamespaceUtils.setReferenceIfAttributeDefined(builder, element, "payload-decoder"); - - IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element, - "auto-commit", "autoCommitOffset"); IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element, "use-context-message-builder", "useMessageBuilderFactory"); IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element, diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaNamespaceHandler.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaNamespaceHandler.java index d8e1d7a2c3..e44cc21158 100644 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaNamespaceHandler.java +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaNamespaceHandler.java @@ -1,5 +1,5 @@ /* - * Copyright 2002-2015 the original author or authors. + * Copyright 2002-2016 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -22,19 +22,15 @@ import org.springframework.integration.config.xml.AbstractIntegrationNamespaceHa * The namespace handler for the Kafka namespace * * @author Soby Chacko + * @author Gary Russell * @since 0.5 * */ public class KafkaNamespaceHandler extends AbstractIntegrationNamespaceHandler { @Override - @SuppressWarnings("deprecation") public void init() { - registerBeanDefinitionParser("zookeeper-connect", new ZookeeperConnectParser()); - registerBeanDefinitionParser("inbound-channel-adapter", new KafkaInboundChannelAdapterParser()); registerBeanDefinitionParser("outbound-channel-adapter", new KafkaOutboundChannelAdapterParser()); - registerBeanDefinitionParser("producer-context", new KafkaProducerContextParser()); - registerBeanDefinitionParser("consumer-context", new KafkaConsumerContextParser()); registerBeanDefinitionParser("message-driven-channel-adapter", new KafkaMessageDrivenChannelAdapterParser()); } diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaOutboundChannelAdapterParser.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaOutboundChannelAdapterParser.java index 07e6182462..1781101b86 100644 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaOutboundChannelAdapterParser.java +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaOutboundChannelAdapterParser.java @@ -1,5 +1,5 @@ /* - * Copyright 2013-2014 the original author or authors. + * Copyright 2013-2016 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -13,6 +13,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.springframework.integration.kafka.config.xml; import org.w3c.dom.Element; @@ -24,12 +25,12 @@ import org.springframework.beans.factory.xml.ParserContext; import org.springframework.integration.config.xml.AbstractOutboundChannelAdapterParser; import org.springframework.integration.config.xml.IntegrationNamespaceUtils; import org.springframework.integration.kafka.outbound.KafkaProducerMessageHandler; -import org.springframework.util.StringUtils; /** * * @author Soby Chacko * @author Artem Bilan + * @author Gary Russell * @since 0.5 * */ @@ -40,11 +41,9 @@ public class KafkaOutboundChannelAdapterParser extends AbstractOutboundChannelAd final BeanDefinitionBuilder kafkaProducerMessageHandlerBuilder = BeanDefinitionBuilder.genericBeanDefinition(KafkaProducerMessageHandler.class); - final String kafkaServerBeanName = element.getAttribute("kafka-producer-context-ref"); + final String kafkaTemplateBeanName = element.getAttribute("kafka-template"); - if (StringUtils.hasText(kafkaServerBeanName)) { - kafkaProducerMessageHandlerBuilder.addConstructorArgReference(kafkaServerBeanName); - } + kafkaProducerMessageHandlerBuilder.addConstructorArgReference(kafkaTemplateBeanName); BeanDefinition topicExpressionDef = IntegrationNamespaceUtils.createExpressionDefinitionFromValueOrExpression("topic", "topic-expression", diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaProducerContextParser.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaProducerContextParser.java deleted file mode 100644 index 422e482896..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/KafkaProducerContextParser.java +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Copyright 2002-2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.config.xml; - -import java.util.Map; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.w3c.dom.Element; - -import org.springframework.beans.BeanMetadataElement; -import org.springframework.beans.factory.support.AbstractBeanDefinition; -import org.springframework.beans.factory.support.BeanDefinitionBuilder; -import org.springframework.beans.factory.support.ManagedMap; -import org.springframework.beans.factory.xml.AbstractSimpleBeanDefinitionParser; -import org.springframework.beans.factory.xml.ParserContext; -import org.springframework.integration.config.xml.IntegrationNamespaceUtils; -import org.springframework.integration.kafka.support.KafkaProducerContext; -import org.springframework.integration.kafka.support.ProducerConfiguration; -import org.springframework.integration.kafka.support.ProducerFactoryBean; -import org.springframework.integration.kafka.support.ProducerMetadata; -import org.springframework.integration.kafka.util.EncoderAdaptingSerializer; -import org.springframework.util.Assert; -import org.springframework.util.StringUtils; -import org.springframework.util.xml.DomUtils; - -/** - * @author Soby Chacko - * @author Ilayaperumal Gopinathan - * @author Gary Russell - * @author Artem Bilan - * @since 0.5 - */ -public class KafkaProducerContextParser extends AbstractSimpleBeanDefinitionParser { - - private static final Log log = LogFactory.getLog(KafkaProducerContextParser.class); - - @Override - protected Class getBeanClass(final Element element) { - return KafkaProducerContext.class; - } - - @Override - protected void doParse(final Element element, final ParserContext parserContext, final BeanDefinitionBuilder builder) { - super.doParse(element, parserContext, builder); - - IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element, "phase"); - IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element, "auto-startup"); - - final Element topics = DomUtils.getChildElementByTagName(element, "producer-configurations"); - parseProducerConfigurations(topics, parserContext, builder, element); - } - - - @Override - protected boolean isEligibleAttribute(String attributeName) { - return !"producer-properties".equals(attributeName) && super.isEligibleAttribute(attributeName); - } - - private void parseProducerConfigurations(Element topics, ParserContext parserContext, - BeanDefinitionBuilder builder, Element parentElem) { - Map producerConfigurationsMap = new ManagedMap(); - - for (Element producerConfiguration : DomUtils.getChildElementsByTagName(topics, "producer-configuration")) { - - BeanDefinitionBuilder producerMetadataBuilder = - BeanDefinitionBuilder.genericBeanDefinition(ProducerMetadata.class); - producerMetadataBuilder.addConstructorArgValue(producerConfiguration.getAttribute("topic")); - producerMetadataBuilder.addConstructorArgValue(producerConfiguration.getAttribute("key-class-type")); - producerMetadataBuilder.addConstructorArgValue(producerConfiguration.getAttribute("value-class-type")); - - String keySerializer = producerConfiguration.getAttribute("key-serializer"); - String keyEncoder = producerConfiguration.getAttribute("key-encoder"); - Assert.isTrue((StringUtils.hasText(keySerializer) ^ StringUtils.hasText(keyEncoder)), - "Exactly one of 'key-serializer' or 'key-encoder' must be specified"); - if (StringUtils.hasText(keyEncoder)) { - if (log.isWarnEnabled()) { - log.warn("'key-encoder' is a deprecated option, use 'key-serializer' instead."); - } - BeanDefinitionBuilder encoderAdaptingSerializerBean = BeanDefinitionBuilder.genericBeanDefinition(EncoderAdaptingSerializer.class); - encoderAdaptingSerializerBean.addConstructorArgReference(keyEncoder); - producerMetadataBuilder.addConstructorArgValue(encoderAdaptingSerializerBean.getBeanDefinition()); - } - else { - producerMetadataBuilder.addConstructorArgReference(keySerializer); - } - - String valueSerializer = producerConfiguration.getAttribute("value-serializer"); - String valueEncoder = producerConfiguration.getAttribute("value-encoder"); - Assert.isTrue((StringUtils.hasText(valueSerializer) ^ StringUtils.hasText(valueEncoder)), - "Exactly one of 'value-serializer' or 'value-encoder' must be specified"); - if (StringUtils.hasText(valueEncoder)) { - if (log.isWarnEnabled()) { - log.warn("'value-encoder' is a deprecated option, use 'value-serializer' instead."); - } - BeanDefinitionBuilder encoderAdaptingSerializerBean = - BeanDefinitionBuilder.genericBeanDefinition(EncoderAdaptingSerializer.class); - encoderAdaptingSerializerBean.addConstructorArgReference(valueEncoder); - producerMetadataBuilder.addConstructorArgValue(encoderAdaptingSerializerBean.getBeanDefinition()); - } - else { - producerMetadataBuilder.addConstructorArgReference(valueSerializer); - } - - IntegrationNamespaceUtils.setReferenceIfAttributeDefined(producerMetadataBuilder, producerConfiguration, - "partitioner"); - if (StringUtils.hasText(producerConfiguration.getAttribute("partitioner"))) { - if (log.isWarnEnabled()) { - log.warn("'partitioner' is a deprecated option. Use the 'kafka_partitionId' message header or " + - "the 'partition-id' (or 'partition-id-expression') attribute."); - } - } - IntegrationNamespaceUtils.setValueIfAttributeDefined(producerMetadataBuilder, producerConfiguration, - "compression-type"); - IntegrationNamespaceUtils.setValueIfAttributeDefined(producerMetadataBuilder, producerConfiguration, - "batch-bytes"); - IntegrationNamespaceUtils.setValueIfAttributeDefined(producerMetadataBuilder, producerConfiguration, - "sync"); - IntegrationNamespaceUtils.setValueIfAttributeDefined(producerMetadataBuilder, producerConfiguration, - "send-timeout"); - IntegrationNamespaceUtils.setValueIfAttributeDefined(producerMetadataBuilder, producerConfiguration, - "charset"); - - AbstractBeanDefinition producerMetadataBeanDefinition = producerMetadataBuilder.getBeanDefinition(); - - String producerPropertiesBean = parentElem.getAttribute("producer-properties"); - - BeanDefinitionBuilder producerFactoryBuilder = - BeanDefinitionBuilder.genericBeanDefinition(ProducerFactoryBean.class); - producerFactoryBuilder.addConstructorArgValue(producerMetadataBeanDefinition); - - final String brokerList = producerConfiguration.getAttribute("broker-list"); - if (StringUtils.hasText(brokerList)) { - producerFactoryBuilder.addConstructorArgValue(producerConfiguration.getAttribute("broker-list")); - } - - if (StringUtils.hasText(producerPropertiesBean)) { - producerFactoryBuilder.addConstructorArgReference(producerPropertiesBean); - } - - AbstractBeanDefinition producerFactoryBeanDefinition = producerFactoryBuilder.getBeanDefinition(); - - BeanDefinitionBuilder producerConfigurationBuilder = - BeanDefinitionBuilder.genericBeanDefinition(ProducerConfiguration.class) - .addConstructorArgValue(producerMetadataBeanDefinition) - .addConstructorArgValue(producerFactoryBeanDefinition); - IntegrationNamespaceUtils.setReferenceIfAttributeDefined(producerConfigurationBuilder, producerConfiguration, - "conversion-service"); - IntegrationNamespaceUtils.setReferenceIfAttributeDefined(producerConfigurationBuilder, producerConfiguration, - "producer-listener"); - producerConfigurationsMap.put(producerConfiguration.getAttribute("topic"), - producerConfigurationBuilder.getBeanDefinition()); - } - - builder.addPropertyValue("producerConfigurations", producerConfigurationsMap); - } - -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/ZookeeperConnectParser.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/ZookeeperConnectParser.java deleted file mode 100644 index 3cda6c3b10..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/config/xml/ZookeeperConnectParser.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.kafka.config.xml; - -import org.springframework.beans.factory.support.BeanDefinitionBuilder; -import org.springframework.beans.factory.xml.AbstractSimpleBeanDefinitionParser; -import org.springframework.beans.factory.xml.BeanDefinitionParserDelegate; -import org.springframework.beans.factory.xml.ParserContext; -import org.springframework.integration.config.xml.IntegrationNamespaceUtils; -import org.springframework.integration.kafka.support.ZookeeperConnect; -import org.w3c.dom.Element; - -/** - * @author Soby Chacko - * @since 0.5 - */ -public class ZookeeperConnectParser extends AbstractSimpleBeanDefinitionParser { - @Override - protected Class getBeanClass(final Element element) { - return ZookeeperConnect.class; - } - - @Override - protected void doParse(final Element element, final ParserContext parserContext, final BeanDefinitionBuilder builder) { - super.doParse(element, parserContext, builder); - - IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element, - BeanDefinitionParserDelegate.SCOPE_ATTRIBUTE); - IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element, "zk-connect"); - IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element, "zk-connection-timeout"); - IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element, "zk-session-timeout"); - IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element, "zk-sync-time"); - } -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/AbstractConfiguration.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/AbstractConfiguration.java deleted file mode 100644 index 047b5730d2..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/AbstractConfiguration.java +++ /dev/null @@ -1,187 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.core; - -import java.util.List; - -import org.springframework.beans.factory.InitializingBean; -import org.springframework.util.Assert; -import org.springframework.util.CollectionUtils; -import org.springframework.util.StringUtils; - -/** - * Default implementation of a {@link Configuration}, storing the default topic and partitions, - * as well as connectivity parameters. - * - * Implementors must provide a strategy for retrieving the seed brokers. - * - * @author Marius Bogoevici - */ -public abstract class AbstractConfiguration implements InitializingBean, Configuration { - - private List defaultPartitions; - - private String defaultTopic; - - private String clientId = KafkaConsumerDefaults.GROUP_ID; - - private int minBytes = KafkaConsumerDefaults.MIN_FETCH_BYTES; - - private int maxWait = KafkaConsumerDefaults.MAX_WAIT_TIME_IN_MS; - - private int bufferSize = KafkaConsumerDefaults.SOCKET_BUFFER_SIZE_INT; - - private int socketTimeout = KafkaConsumerDefaults.SOCKET_TIMEOUT_INT; - - private int backoff = KafkaConsumerDefaults.BACKOFF_INCREMENT_INT; - - private int fetchMetadataTimeout = KafkaConsumerDefaults.FETCH_METADATA_TIMEOUT; - - /** - * The minimum amount of data that a server fetch operation will wait for before returning, - * unless {@code maxWait} has elapsed. - * In conjunction with {@link Configuration#getMaxWait()}}, controls latency - * and throughput. - * Smaller values increase responsiveness, but may increase the number of poll operations, - * potentially reducing throughput and increasing CPU consumption. - * @param minBytes the amount of data to fetch - */ - public void setMinBytes(int minBytes) { - this.minBytes = minBytes; - } - - @Override - public int getMinBytes() { - return this.minBytes; - } - - /** - * The maximum amount of time that a server fetch operation will wait before returning - * (unless {@code minFetchSizeInBytes}) are available. - * In conjunction with {@link AbstractConfiguration#setMinBytes(int)}, - * controls latency and throughput. - * Smaller intervals increase responsiveness, but may increase - * the number of poll operations, potentially increasing CPU - * consumption and reducing throughput. - * @param maxWait timeout to wait - */ - public void setMaxWait(int maxWait) { - this.maxWait = maxWait; - } - - @Override - public int getMaxWait() { - return this.maxWait; - } - - @Override - public String getClientId() { - return this.clientId; - } - - /** - * A client name to be used throughout this connection. - * @param clientId the client name - */ - public void setClientId(String clientId) { - this.clientId = clientId; - } - - @Override - public int getBufferSize() { - return this.bufferSize; - } - - /** - * The buffer size for this client - * @param bufferSize the buffer size - */ - public void setBufferSize(int bufferSize) { - this.bufferSize = bufferSize; - } - - @Override - public int getSocketTimeout() { - return this.socketTimeout; - } - - /** - * The socket timeout for this client - * @param socketTimeout the socket timeout - */ - public void setSocketTimeout(int socketTimeout) { - this.socketTimeout = socketTimeout; - } - - @Override - public int getBackOff() { - return backoff; - } - - /** - * The retry backoff time for this client - * @param backoff the retry backoff - */ - public void setBackoff(int backoff) { - this.backoff = backoff; - } - - /** - * The timeout on fetching metadata (e.g. partition leaders) - * @param fetchMetadataTimeout timeout - */ - public void setFetchMetadataTimeout(int fetchMetadataTimeout) { - this.fetchMetadataTimeout = fetchMetadataTimeout; - } - - @Override - public int getFetchMetadataTimeout() { - return this.fetchMetadataTimeout; - } - - @Override - public final List getBrokerAddresses() { - return doGetBrokerAddresses(); - } - - @Override - public List getDefaultPartitions() { - return this.defaultPartitions; - } - - public void setDefaultPartitions(List defaultPartitions) { - this.defaultPartitions = defaultPartitions; - } - - @Override - public String getDefaultTopic() { - return this.defaultTopic; - } - - public void setDefaultTopic(String defaultTopic) { - this.defaultTopic = defaultTopic; - } - - @Override - public void afterPropertiesSet() throws Exception { - Assert.isTrue(CollectionUtils.isEmpty(this.defaultPartitions) || StringUtils.isEmpty(this.defaultTopic) - , "A list of default partitions or a default topic may be specified, but not both"); - } - - protected abstract List doGetBrokerAddresses(); - -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/BrokerAddress.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/BrokerAddress.java deleted file mode 100644 index f81dab2df7..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/BrokerAddress.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package org.springframework.integration.kafka.core; - -import org.springframework.util.Assert; - -import kafka.cluster.Broker; - -/** - * Encapsulates the address of a Kafka broker. - * - * @author Marius Bogoevici - */ -public class BrokerAddress { - - public static final int DEFAULT_PORT = 9092; - - private final String host; - - private final int port; - - public BrokerAddress(String host, int port) { - Assert.hasText(host, "Host cannot be empty"); - this.host = host; - this.port = port; - } - - public BrokerAddress(String host) { - this(host, DEFAULT_PORT); - } - - public BrokerAddress(Broker broker) { - Assert.notNull(broker, "Broker cannot be null"); - this.host = broker.host(); - this.port = broker.port(); - } - - public static BrokerAddress fromAddress(String address) { - String[] split = address.split(":"); - if (split.length == 0 || split.length > 2) { - throw new IllegalArgumentException("Expected format [:]"); - } - if (split.length == 2) { - return new BrokerAddress(split[0], Integer.parseInt(split[1])); - } - else { - return new BrokerAddress(split[0]); - } - } - - public String getHost() { - return this.host; - } - - public int getPort() { - return this.port; - } - - @Override - public int hashCode() { - return 31 * this.host.hashCode() + this.port; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - - BrokerAddress brokerAddress = (BrokerAddress) o; - - return this.port == brokerAddress.port && this.host.equals(brokerAddress.host); - } - - @Override - public String toString() { - return this.host + ":" + this.port; - } - -} - - diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/BrokerAddressListConfiguration.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/BrokerAddressListConfiguration.java deleted file mode 100644 index d013b09d75..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/BrokerAddressListConfiguration.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package org.springframework.integration.kafka.core; - -import java.util.Arrays; -import java.util.List; - -/** - * Kafka {@link Configuration} where the seed brokers are set up explicitly. - * - * @author Marius Bogoevici - */ -public class BrokerAddressListConfiguration extends AbstractConfiguration { - - private final List brokerAddresses; - - public BrokerAddressListConfiguration(BrokerAddress... brokerAddresses) { - this.brokerAddresses = Arrays.asList(brokerAddresses); - } - - @Override - protected List doGetBrokerAddresses() { - return brokerAddresses; - } - -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/Configuration.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/Configuration.java deleted file mode 100644 index da4342ec49..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/Configuration.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.core; - -import java.util.List; - -/** - * Used to configure a {@link DefaultConnectionFactory}. Provides a list of seed brokers. - * - * @author Marius Bogoevici - */ -public interface Configuration { - - /** - * The minimum amount of data that a server fetch operation will wait for before returning, - * unless {@code maxWait} has elapsed. - * In conjunction with {@link Configuration#getMaxWait()}}, controls latency - * and throughput. - * Smaller values increase responsiveness, but may increase the number of poll operations, - * potentially reducing throughput and increasing CPU consumption. - * @return the minimum amount of data for a fetch operation - */ - int getMinBytes(); - - /** - * The maximum amount of time that a server fetch operation will wait before returning - * (unless {@code minFetchSizeInBytes}) are available. - * In conjunction with {@link AbstractConfiguration#setMinBytes(int)}, - * controls latency and throughput. - * Smaller intervals increase responsiveness, but may increase - * the number of poll operations, potentially increasing CPU - * consumption and reducing throughput. - * @return the maximum wait time for a fetch operation - */ - int getMaxWait(); - - /** - * The client name to be used throughout this connection. - * @return the client id for a connection - */ - String getClientId(); - - /** - * The buffer size for this client - * @return the buffer size - */ - int getBufferSize(); - - /** - * The socket timeout for this client - * @return the socket timeout - */ - int getSocketTimeout(); - - /** - * The retry backoff for this client - * @return the retry backoff - * @since 1.3 - */ - int getBackOff(); - - /** - * The timeout on fetching metadata (e.g. partition leaders) - * @return the fetch metadata timeout - */ - int getFetchMetadataTimeout(); - - /** - * The list of seed broker addresses used by this Configuration. - * @return the broker addresses - */ - List getBrokerAddresses(); - - /** - * A list of default partitions to perform operations on. - * @return the list of partitions - */ - List getDefaultPartitions(); - - /** - * A default topic to perform operations on. - * @return a topic name - */ - String getDefaultTopic(); - - -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/Connection.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/Connection.java deleted file mode 100644 index b7cb86c9e4..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/Connection.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package org.springframework.integration.kafka.core; - -import java.util.Map; - -import kafka.api.OffsetRequest; - -/** - * A connection to a Kafka broker. - * - * @author Marius Bogoevici - */ -public interface Connection { - - /** - * Fetch data from a Kafka broker. - * @param fetchRequests a list of fetch operations - * @return message batches, indexed by partition - * @throws ConsumerException the ConsumerException if any underlying error - */ - Result fetch(FetchRequest... fetchRequests) throws ConsumerException; - - /** - * Fetch an actual offset in the partition, immediately before the given reference time, - * or the smallest and largest value, respectively, if the special values -1 - * ({@link OffsetRequest#LatestTime()}) and -2 ({@link OffsetRequest#EarliestTime()}) - * are used . To be used to position the initial offset of a read operation. - * @param referenceTime The returned values will be before this time, if they exist. The special - * values -2 ({@link OffsetRequest#EarliestTime()}) and -1 ({@link OffsetRequest#LatestTime()}) are supported. - * @param partitions the offsets, indexed by {@link Partition} - * @return any errors, an empty {@link Result} in case of success - * @throws ConsumerException the ConsumerException if any underlying error - */ - Result fetchInitialOffset(long referenceTime, Partition... partitions) throws ConsumerException; - - /** - * Fetch offsets from the native Kafka offset management system. - * @param consumerId the id of the consumer - * @param partitions the list of partitions whose offsets are queried for - * @return any errors, an empty {@link Result} in case of success - * @throws ConsumerException the ConsumerException if any underlying error - */ - Result fetchStoredOffsetsForConsumer(String consumerId, Partition... partitions) throws ConsumerException; - - /** - * Update offsets in the native Kafka offset management system. - * @param consumerId the id of the consumer - * @param offsets the offsets, indexed by {@link Partition} - * @return any errors, an empty {@link Result} in case of success - * @throws ConsumerException the ConsumerException if any underlying error - */ - Result commitOffsetsForConsumer(String consumerId, Map offsets) throws ConsumerException; - - /** - * Retrieve the leader broker addresses for all the partitions in the given topics. - * @param topics the topics whose partitions we query for - * @return broker addresses, indexed by {@link Partition} - * @throws ConsumerException the ConsumerException if any underlying error - * @deprecated as of 1.3, only {@link ConnectionFactory#getLeaders(Iterable)} should be used - */ - @Deprecated - Result findLeaders(String... topics) throws ConsumerException; - - - /** - * The broker address for this consumer - * @return broker address - */ - BrokerAddress getBrokerAddress(); - - /** - * Closes the connection to the broker. No further operations are permitted. - */ - void close(); - -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ConnectionFactory.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ConnectionFactory.java deleted file mode 100644 index a52707c66f..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ConnectionFactory.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package org.springframework.integration.kafka.core; - -import java.util.Collection; -import java.util.Map; - -/** - * Creates Kafka connections and retrieves metadata for topics and partitions. - * - * @author Marius Bogoevici - * - */ -public interface ConnectionFactory { - - /** - * Create a connection to a Kafka broker, caching it internally - * @param brokerAddress a broker address - * @return a working connection - */ - Connection connect(BrokerAddress brokerAddress); - - /** - * Close the connection to the broker - * @param brokerAddress brokerAddress - */ - void disconnect(BrokerAddress brokerAddress); - - /** - * Retrieve the leaders for a set of partitions. - * @param partitions whose leaders are queried - * @return the broker associated with the provided topic and partition - */ - Map getLeaders(Iterable partitions); - - /** - * Return the leader for a single partition - * @param partition the partition whose leader is queried - * @return the leader's address - */ - BrokerAddress getLeader(Partition partition); - - /** - * Refresh the cached metadata (i.e. leader topology and partitions). To be called when the topology changes - * are detected (i.e. brokers leave and/or partition leaders change) and that results in fetch errors, - * for instance. - * @param topics the topics for which to refresh the leaders - */ - void refreshMetadata(Collection topics); - - /** - * Retrieves the partitions of a given topic - * @param topic the topic to query for - * @return a list of partitions - */ - Collection getPartitions(String topic); - -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ConsumerException.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ConsumerException.java deleted file mode 100644 index 49d9698990..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ConsumerException.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package org.springframework.integration.kafka.core; - -import kafka.javaapi.consumer.SimpleConsumer; - -/** - * Wraps exceptions thrown by {@link SimpleConsumer} calls. - * - * Because the client is originally written in Scala, checked exceptions are missing from the method signatures - * and cannot be caught directly. - * - * @author Marius Bogoevici - */ -@SuppressWarnings("serial") -public class ConsumerException extends RuntimeException { - - public ConsumerException(Throwable cause) { - super(cause); - } - - public ConsumerException(String message) { - super(message); - } - -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/DefaultConnection.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/DefaultConnection.java deleted file mode 100644 index fed875d8ab..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/DefaultConnection.java +++ /dev/null @@ -1,324 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.core; - -import java.util.Arrays; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Random; -import java.util.concurrent.atomic.AtomicInteger; - -import com.gs.collections.api.block.function.Function; -import com.gs.collections.api.block.function.Function2; -import com.gs.collections.api.tuple.Pair; -import com.gs.collections.impl.list.mutable.FastList; -import com.gs.collections.impl.tuple.Tuples; -import com.gs.collections.impl.utility.LazyIterate; -import com.gs.collections.impl.utility.MapIterate; -import kafka.api.FetchRequestBuilder; -import kafka.api.PartitionOffsetRequestInfo; -import kafka.cluster.Broker; -import kafka.common.ErrorMapping; -import kafka.common.OffsetAndMetadata; -import kafka.common.OffsetMetadataAndError; -import kafka.common.TopicAndPartition; -import kafka.javaapi.FetchResponse; -import kafka.javaapi.OffsetCommitRequest; -import kafka.javaapi.OffsetCommitResponse; -import kafka.javaapi.OffsetFetchRequest; -import kafka.javaapi.OffsetFetchResponse; -import kafka.javaapi.OffsetRequest; -import kafka.javaapi.OffsetResponse; -import kafka.javaapi.PartitionMetadata; -import kafka.javaapi.TopicMetadata; -import kafka.javaapi.TopicMetadataRequest; -import kafka.javaapi.TopicMetadataResponse; -import kafka.javaapi.consumer.SimpleConsumer; -import kafka.javaapi.message.ByteBufferMessageSet; -import kafka.message.MessageAndOffset; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import org.springframework.util.Assert; - -/** - * A connection to a Kafka broker. - * - * @author Marius Bogoevici - */ -public class DefaultConnection implements Connection { - - private static Log log = LogFactory.getLog(DefaultConnection.class); - - private final AtomicInteger correlationIdCounter = new AtomicInteger(new Random(new Date().getTime()).nextInt()); - - private final SimpleConsumer simpleConsumer; - - private final BrokerAddress brokerAddress; - - private int minBytes; - - private int maxWait; - - public DefaultConnection(BrokerAddress brokerAddress, String clientId, int bufferSize, int soTimeout, - int minBytes, int maxWait) { - this.brokerAddress = brokerAddress; - this.minBytes = minBytes; - this.maxWait = maxWait; - this.simpleConsumer = - new SimpleConsumer(brokerAddress.getHost(), brokerAddress.getPort(), soTimeout, bufferSize, clientId); - } - - /** - * @see Connection#getBrokerAddress() - */ - @Override - public BrokerAddress getBrokerAddress() { - return brokerAddress; - } - - @Override - public void close() { - this.simpleConsumer.close(); - } - - /** - * Fetche data from Kafka. - * @return a combination of messages and errors, depending on whether the invocation was successful or not - * @throws ConsumerException the ConsumerException if any underlying error - */ - @Override - public Result fetch(FetchRequest... requests) throws ConsumerException { - FetchRequestBuilder fetchRequestBuilder = new FetchRequestBuilder(); - for (FetchRequest request : requests) { - Partition partition = request.getPartition(); - long offset = request.getOffset(); - int maxSize = request.getMaxSizeInBytes(); - fetchRequestBuilder.addFetch(partition.getTopic(), partition.getId(), offset, maxSize); - } - FetchResponse fetchResponse; - try { - fetchResponse = this.simpleConsumer.fetch(fetchRequestBuilder.maxWait(maxWait).minBytes(minBytes).build()); - } - catch (Exception e) { - throw new ConsumerException(e); - } - ResultBuilder resultBuilder = new ResultBuilder(); - for (final FetchRequest request : requests) { - Partition partition = request.getPartition(); - if (log.isTraceEnabled()) { - log.trace("Reading from " + partition + "@" + request.getOffset()); - } - short errorCode = fetchResponse.errorCode(partition.getTopic(), partition.getId()); - if (ErrorMapping.NoError() == errorCode) { - ByteBufferMessageSet messageSet = fetchResponse.messageSet(partition.getTopic(), partition.getId()); - List kafkaMessages = LazyIterate.collect(messageSet, - new ConvertToKafkaMessageFunction(request)).toList(); - long highWatermark = fetchResponse.highWatermark(partition.getTopic(), partition.getId()); - resultBuilder.add(partition).withResult(new KafkaMessageBatch(partition, kafkaMessages, highWatermark)); - } - else { - resultBuilder.add(partition).withError(errorCode); - } - } - return resultBuilder.build(); - } - - @Override - public Result fetchStoredOffsetsForConsumer(String consumerId, Partition... partitions) throws - ConsumerException { - FastList topicsAndPartitions = FastList.newList(Arrays.asList(partitions)) - .collect(new ConvertToTopicAndPartitionFunction()); - OffsetFetchRequest offsetFetchRequest = new OffsetFetchRequest(consumerId, topicsAndPartitions, - kafka.api.OffsetFetchRequest.CurrentVersion(), createCorrelationId(), simpleConsumer.clientId()); - OffsetFetchResponse offsetFetchResponse = null; - try { - offsetFetchResponse = simpleConsumer.fetchOffsets(offsetFetchRequest); - } - catch (Exception e) { - throw new ConsumerException(e); - } - ResultBuilder resultBuilder = new ResultBuilder(); - for (Partition partition : partitions) { - OffsetMetadataAndError offsetMetadataAndError - = offsetFetchResponse.offsets().get(new TopicAndPartition(partition.getTopic(), partition.getId())); - short errorCode = offsetMetadataAndError.error(); - if (ErrorMapping.NoError() == errorCode) { - resultBuilder.add(partition).withResult(offsetMetadataAndError.offset()); - } - else { - resultBuilder.add(partition).withError(errorCode); - } - } - return resultBuilder.build(); - } - - /** - * @see Connection#fetchInitialOffset(long, Partition...) - */ - @Override - public Result fetchInitialOffset(long referenceTime, Partition... partitions) throws - ConsumerException { - Assert.isTrue(partitions.length > 0, "Must provide at least one partition"); - Map infoMap = new HashMap(); - for (Partition partition : partitions) { - infoMap.put(new TopicAndPartition(partition.getTopic(), partition.getId()), - new PartitionOffsetRequestInfo(referenceTime, 1)); - } - OffsetRequest offsetRequest = - new OffsetRequest(infoMap, kafka.api.OffsetRequest.CurrentVersion(), simpleConsumer.clientId()); - OffsetResponse offsetResponse = null; - try { - offsetResponse = simpleConsumer.getOffsetsBefore(offsetRequest); - } - catch (Exception e) { - throw new ConsumerException(e); - } - ResultBuilder resultBuilder = new ResultBuilder(); - for (Partition partition : partitions) { - short errorCode = offsetResponse.errorCode(partition.getTopic(), partition.getId()); - if (ErrorMapping.NoError() == errorCode) { - long[] offsets = offsetResponse.offsets(partition.getTopic(), partition.getId()); - if (offsets.length == 0) { - // normally, we shouldn't get here - either an offset has been returned, - // or an error. However, in case something went wrong, the check protects against an - // ArrayIndexOutOfBoundsException - throw new ConsumerException("Inconsistent response: no error has been returned, " + - "but no offsets either"); - } - resultBuilder.add(partition).withResult(offsets[0]); - } - else { - resultBuilder.add(partition).withError(errorCode); - } - } - return resultBuilder.build(); - } - - @Override - public Result commitOffsetsForConsumer(String consumerId, Map offsets) - throws ConsumerException { - Map requestInfo = - MapIterate.collect(offsets, new CreateRequestInfoMapEntryFunction()); - OffsetCommitResponse offsetCommitResponse = null; - try { - offsetCommitResponse = simpleConsumer.commitOffsets( - new OffsetCommitRequest(consumerId, requestInfo, createCorrelationId(), - simpleConsumer.clientId(), kafka.api.OffsetCommitRequest.CurrentVersion())); - } - catch (Exception e) { - throw new ConsumerException(e); - } - ResultBuilder resultBuilder = new ResultBuilder(); - for (TopicAndPartition topicAndPartition : requestInfo.keySet()) { - if (offsetCommitResponse.errors().containsKey(topicAndPartition)) { - Partition partition = new Partition(topicAndPartition.topic(), topicAndPartition.partition()); - resultBuilder.add(partition).withError((Short) offsetCommitResponse.errors().get(topicAndPartition)); - } - } - return resultBuilder.build(); - } - - /** - * @see Connection#findLeaders(String...) - */ - @Override - @Deprecated - public Result findLeaders(String... topics) throws ConsumerException { - TopicMetadataRequest topicMetadataRequest = - new TopicMetadataRequest(Arrays.asList(topics), createCorrelationId()); - TopicMetadataResponse topicMetadataResponse = null; - try { - topicMetadataResponse = simpleConsumer.send(topicMetadataRequest); - } - catch (Exception e) { - throw new ConsumerException(e); - } - ResultBuilder resultBuilder = new ResultBuilder(); - for (TopicMetadata topicMetadata : topicMetadataResponse.topicsMetadata()) { - if (topicMetadata.errorCode() != ErrorMapping.NoError()) { - resultBuilder.add(new Partition(topicMetadata.topic(), -1)).withError(topicMetadata.errorCode()); - } - else { - for (PartitionMetadata partitionMetadata : topicMetadata.partitionsMetadata()) { - Partition partition = new Partition(topicMetadata.topic(), partitionMetadata.partitionId()); - if (ErrorMapping.NoError() == partitionMetadata.errorCode()) { - Broker leader = partitionMetadata.leader(); - BrokerAddress result = new BrokerAddress(leader.host(), leader.port()); - resultBuilder.add(partition).withResult(result); - } - else { - resultBuilder.add(partition).withError(partitionMetadata.errorCode()); - } - } - - } - } - return resultBuilder.build(); - } - - - - /** - * Creat a pseudo-unique correlation id for requests and responses - * @return correlation id - */ - private Integer createCorrelationId() { - return correlationIdCounter.incrementAndGet(); - } - - @SuppressWarnings("serial") - private static class ConvertToKafkaMessageFunction implements Function { - private final FetchRequest request; - - public ConvertToKafkaMessageFunction(FetchRequest request) { - this.request = request; - } - - @Override - public KafkaMessage valueOf(MessageAndOffset messageAndOffset) { - return new KafkaMessage(messageAndOffset.message(), - new KafkaMessageMetadata(request.getPartition(), messageAndOffset.offset(), - messageAndOffset.nextOffset())); - } - } - - @SuppressWarnings("serial") - private static class ConvertToTopicAndPartitionFunction implements Function { - - @Override - public TopicAndPartition valueOf(Partition partition) { - return new TopicAndPartition(partition.getTopic(), partition.getId()); - } - - } - - @SuppressWarnings("serial") - private static class CreateRequestInfoMapEntryFunction - implements Function2> { - - @Override - public Pair value(Partition partition, Long offset) { - return Tuples.pair(new TopicAndPartition(partition.getTopic(), partition.getId()), - new OffsetAndMetadata(offset, OffsetAndMetadata.NoMetadata(), OffsetAndMetadata.InvalidTime())); - } - - } - -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/DefaultConnectionFactory.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/DefaultConnectionFactory.java deleted file mode 100644 index b7aa3c039b..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/DefaultConnectionFactory.java +++ /dev/null @@ -1,276 +0,0 @@ -/* - * Copyright 2015-2016 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.core; - -import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.Map; -import java.util.concurrent.atomic.AtomicReference; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import org.springframework.beans.factory.DisposableBean; -import org.springframework.beans.factory.InitializingBean; -import org.springframework.util.Assert; - -import com.gs.collections.api.block.function.Function; -import com.gs.collections.api.block.predicate.Predicate; -import com.gs.collections.api.partition.PartitionIterable; -import com.gs.collections.impl.block.factory.Functions; -import com.gs.collections.impl.map.mutable.UnifiedMap; -import com.gs.collections.impl.utility.Iterate; -import com.gs.collections.impl.utility.ListIterate; - -import kafka.client.ClientUtils$; -import kafka.cluster.Broker; -import kafka.common.ErrorMapping; -import kafka.javaapi.TopicMetadata; -import kafka.javaapi.TopicMetadataResponse; -import scala.collection.JavaConversions; -import scala.collection.Seq; - -/** - * Default implementation of {@link ConnectionFactory} - * - * @author Marius Bogoevici - * @author Artem Bilan - */ -public class DefaultConnectionFactory implements InitializingBean, ConnectionFactory, DisposableBean { - - private final static Log log = LogFactory.getLog(DefaultConnectionFactory.class); - - public static final Predicate errorlessTopicMetadataPredicate = new ErrorlessTopicMetadataPredicate(); - - private final GetBrokersByPartitionFunction getBrokersByPartitionFunction = new GetBrokersByPartitionFunction(); - - private final Configuration configuration; - - private final AtomicReference metadataCacheHolder = new AtomicReference( - new MetadataCache(Collections.emptySet())); - - private final ReadWriteLock lock = new ReentrantReadWriteLock(); - - private final UnifiedMap kafkaBrokersCache = UnifiedMap.newMap(); - - public DefaultConnectionFactory(Configuration configuration) { - this.configuration = configuration; - } - - public Configuration getConfiguration() { - return this.configuration; - } - - @Override - public void afterPropertiesSet() throws Exception { - Assert.notNull(this.configuration, "Kafka configuration cannot be empty"); - } - - @Override - public void destroy() throws Exception { - for (Connection connection : this.kafkaBrokersCache) { - connection.close(); - } - } - - /** - * @see ConnectionFactory#getLeaders(Iterable) - */ - @Override - public Map getLeaders(Iterable partitions) { - return Iterate.toMap(partitions, Functions.getPassThru(), getBrokersByPartitionFunction); - } - - /** - * @see ConnectionFactory#getLeader(Partition) - */ - @Override - public BrokerAddress getLeader(Partition partition) { - BrokerAddress leader = null; - try { - this.lock.readLock().lock(); - leader = getMetadataCache().getLeader(partition); - } - finally { - this.lock.readLock().unlock(); - } - if (leader == null) { - try { - this.lock.writeLock().lock(); - // double lock check - leader = getMetadataCache().getLeader(partition); - if (leader == null) { - refreshMetadata(Collections.singleton(partition.getTopic())); - leader = getMetadataCache().getLeader(partition); - } - } - finally { - this.lock.writeLock().unlock(); - } - } - if (leader == null) { - throw new PartitionNotFoundException(partition); - } - return leader; - } - - /** - * @see ConnectionFactory#connect(BrokerAddress) - */ - @Override - public Connection connect(BrokerAddress brokerAddress) { - Connection connection = null; - try { - this.lock.readLock().lock(); - connection = this.kafkaBrokersCache.get(brokerAddress); - } - finally { - this.lock.readLock().unlock(); - } - if (connection == null) { - try { - this.lock.writeLock().lock(); - connection = this.kafkaBrokersCache.get(brokerAddress); - if (connection == null) { - connection = new DefaultConnection(brokerAddress, - DefaultConnectionFactory.this.configuration.getClientId(), - DefaultConnectionFactory.this.configuration.getBufferSize(), - DefaultConnectionFactory.this.configuration.getSocketTimeout(), - DefaultConnectionFactory.this.configuration.getMinBytes(), - DefaultConnectionFactory.this.configuration.getMaxWait()); - kafkaBrokersCache.put(brokerAddress, connection); - } - } - finally { - this.lock.writeLock().unlock(); - } - } - return connection; - } - - /** - * @see ConnectionFactory#refreshMetadata(Collection) - */ - @Override - public void refreshMetadata(Collection topics) { - try { - this.lock.writeLock().lock(); - String brokerAddressesAsString = ListIterate - .collect(this.configuration.getBrokerAddresses(), Functions.getToString()).makeString(","); - Seq brokers = null; - try { - brokers = ClientUtils$.MODULE$.parseBrokerList(brokerAddressesAsString); - } - catch (Exception e) { - throw new IllegalStateException("Can not parse Kafka Brokers for: [" + brokerAddressesAsString + "]", e); - } - TopicMetadataResponse topicMetadataResponse = new TopicMetadataResponse(ClientUtils$.MODULE$ - .fetchTopicMetadata(JavaConversions.asScalaSet(new HashSet<>(topics)), - brokers, - this.configuration.getClientId(), this.configuration.getFetchMetadataTimeout(), 0)); - PartitionIterable selectWithoutErrors = Iterate - .partition(topicMetadataResponse.topicsMetadata(), errorlessTopicMetadataPredicate); - this.metadataCacheHolder.set(this.metadataCacheHolder.get().merge(selectWithoutErrors.getSelected())); - if (log.isInfoEnabled()) { - for (TopicMetadata topicMetadata : selectWithoutErrors.getRejected()) { - log.info(String.format("No metadata could be retrieved for '%s'", topicMetadata.topic()), - ErrorMapping.exceptionFor(topicMetadata.errorCode())); - } - } - } - finally { - this.lock.writeLock().unlock(); - } - } - - @Override - public void disconnect(BrokerAddress brokerAddress) { - try { - this.lock.writeLock().lock(); - Connection connection = this.kafkaBrokersCache.get(brokerAddress); - if (connection != null) { - connection.close(); - } - } - finally { - this.lock.writeLock().unlock(); - } - } - - /** - * @see ConnectionFactory#getPartitions(String) - */ - @Override - public Collection getPartitions(String topic) { - // first, we try to read the topic from the cache. We use the read lock to block if a write is in progress - Collection returnedPartitions = null; - try { - this.lock.readLock().lock(); - returnedPartitions = getMetadataCache().getPartitions(topic); - } - finally { - this.lock.readLock().unlock(); - } - // if we got here, it means that the data was not available, we should try a refresh. The lock is reentrant - // so we will not block ourselves - if (returnedPartitions == null) { - try { - this.lock.writeLock().lock(); - // double lock check - returnedPartitions = getMetadataCache().getPartitions(topic); - if (returnedPartitions == null) { - refreshMetadata(Collections.singleton(topic)); - // if data is not available after refreshing, it means that the topic was not found - returnedPartitions = getMetadataCache().getPartitions(topic); - } - } - finally { - lock.writeLock().unlock(); - } - } - if (returnedPartitions == null) { - throw new TopicNotFoundException(topic); - } - return returnedPartitions; - } - - private MetadataCache getMetadataCache() { - return this.metadataCacheHolder.get(); - } - - @SuppressWarnings("serial") - private static class ErrorlessTopicMetadataPredicate implements Predicate { - @Override - public boolean accept(TopicMetadata topicMetadata) { - return topicMetadata.errorCode() == ErrorMapping.NoError(); - } - } - - @SuppressWarnings("serial") - private class GetBrokersByPartitionFunction implements Function { - - @Override - public BrokerAddress valueOf(Partition partition) { - return metadataCacheHolder.get().getLeader(partition); - } - - } - -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/FetchRequest.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/FetchRequest.java deleted file mode 100644 index a3b523f9e6..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/FetchRequest.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.core; - -/** - * Encapsulates a request for fetching messages from the server. - * - * @author Marius Bogoevici - */ -public class FetchRequest { - - private Partition partition; - - private long offset; - - private int maxSizeInBytes; - - public FetchRequest(Partition Partition, long offset, int maxSizeInBytes) { - this.partition = Partition; - this.offset = offset; - this.maxSizeInBytes = maxSizeInBytes; - } - - public Partition getPartition() { - return partition; - } - - public void setPartition(Partition partition) { - this.partition = partition; - } - - public long getOffset() { - return offset; - } - - public void setOffset(long offset) { - this.offset = offset; - } - - public int getMaxSizeInBytes() { - return maxSizeInBytes; - } - - public void setMaxSizeInBytes(int maxSizeInBytes) { - this.maxSizeInBytes = maxSizeInBytes; - } - -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaConsumerDefaults.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaConsumerDefaults.java deleted file mode 100644 index 6953064cbb..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaConsumerDefaults.java +++ /dev/null @@ -1,70 +0,0 @@ -/* - * Copyright 2002-2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.core; - -import kafka.api.OffsetRequest; - -/** - * Kafka adapter specific message headers. - * - * @author Soby Chacko - * @author Marius Bogoevici - * @since 0.5 - */ -public abstract class KafkaConsumerDefaults { - - //High level consumer - public static final String GROUP_ID = "groupid"; - - public static final int SOCKET_TIMEOUT_INT = 30000; - - public static final String SOCKET_TIMEOUT = Integer.toString(SOCKET_TIMEOUT_INT); - - public static final int SOCKET_BUFFER_SIZE_INT = 64*1024; - - public static final String SOCKET_BUFFER_SIZE = Integer.toString(SOCKET_BUFFER_SIZE_INT); - - public static final int FETCH_SIZE_INT = 300 * 1024; - - public static final String FETCH_SIZE = Integer.toString(FETCH_SIZE_INT); - - public static final int BACKOFF_INCREMENT_INT = 1000; - - public static final String BACKOFF_INCREMENT = Integer.toString(BACKOFF_INCREMENT_INT); - - public static final String QUEUED_CHUNKS_MAX = "100"; - - public static final String AUTO_COMMIT_ENABLE = "true"; - - public static final String AUTO_COMMIT_INTERVAL = "10000"; - - public static final String AUTO_OFFSET_RESET = "smallest"; - - //Overriding the default value of -1, which will make the consumer to wait indefinitely - public static final String CONSUMER_TIMEOUT = "5000"; - - public static final String REBALANCE_RETRIES_MAX = "4"; - - public static final int MIN_FETCH_BYTES = 1; - - public static final int MAX_WAIT_TIME_IN_MS = 100; - - public static final long DEFAULT_OFFSET_RESET = OffsetRequest.EarliestTime(); - - public static final int FETCH_METADATA_TIMEOUT = 10000; - -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaMessage.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaMessage.java deleted file mode 100644 index cbf8d19851..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaMessage.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.core; - -import kafka.message.Message; - -import org.springframework.util.ObjectUtils; - -/** - * Wrapper around a {@link Message kafka message} and its {@link KafkaMessageMetadata metadata}. - * - * @author Marius Bogoevici - */ -public class KafkaMessage { - - private final Message message; - - private final KafkaMessageMetadata metadata; - - public KafkaMessage(Message message, KafkaMessageMetadata metadata) { - this.message = message; - this.metadata = metadata; - } - - public Message getMessage() { - return message; - } - - public KafkaMessageMetadata getMetadata() { - return metadata; - } - - @Override - public String toString() { - return "KafkaMessage [" + ObjectUtils.nullSafeToString(message) + ", " - + ObjectUtils.nullSafeToString(metadata.toString()) + "]"; - } - -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaMessageBatch.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaMessageBatch.java deleted file mode 100644 index 8e718cf0b0..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaMessageBatch.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.core; - -import java.util.List; - -/** - * A group of {@link KafkaMessage}s retrieved by a fetch operation - * - * @author Marius Bogoevici - */ -public class KafkaMessageBatch { - - private Partition partition; - - private List messages; - - private long highWatermark; - - public KafkaMessageBatch(Partition partition, List messages, long highWatermark) { - this.partition = partition; - this.messages = messages; - this.highWatermark = highWatermark; - } - - public Partition getPartition() { - return partition; - } - - public void setPartition(Partition partition) { - this.partition = partition; - } - - public List getMessages() { - return messages; - } - - public void setMessages(List messages) { - this.messages = messages; - } - - public long getHighWatermark() { - return highWatermark; - } - - public void setHighWatermark(long highWatermark) { - this.highWatermark = highWatermark; - } - -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaMessageMetadata.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaMessageMetadata.java deleted file mode 100644 index aac33c6050..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaMessageMetadata.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.core; - -import kafka.message.Message; - -import org.springframework.util.Assert; - -/** - * Metadata for a Kafka {@link Message}. - * - * @author Marius Bogoevici - */ -public class KafkaMessageMetadata { - - private final long offset; - - private final long nextOffset; - - private final Partition partition; - - public KafkaMessageMetadata(Partition partition, long offset, long nextOffset) { - Assert.notNull(partition); - Assert.isTrue(offset >= 0); - Assert.isTrue(nextOffset >= 0); - this.offset = offset; - this.nextOffset = nextOffset; - this.partition = partition; - } - - public Partition getPartition() { - return partition; - } - - public long getOffset() { - return offset; - } - - public long getNextOffset() { - return nextOffset; - } - - @Override - public String toString() { - return "KafkaMessageMetadata [" + - "offset=" + offset + - ", nextOffset=" + nextOffset + - ", " + partition.toString(); - } - -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaOperations.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaOperations.java deleted file mode 100644 index 4d6a198c35..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaOperations.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.core; - -/** - * @author Marius Bogoevici - */ -public interface KafkaOperations { - - /** - * Receive data from the server. - * @param messageFetchRequests the set of FetchRequest - * @return the Result with KafkaMessageBatch - */ - Result receive(Iterable messageFetchRequests); - -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaTemplate.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaTemplate.java deleted file mode 100644 index 659abb1a87..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/KafkaTemplate.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.core; - -import com.gs.collections.api.block.function.Function; -import com.gs.collections.api.list.MutableList; -import com.gs.collections.impl.list.mutable.FastList; - - -/** - * A template for executing high-level operations on a set of Kafka brokers. - * - * @author Marius Bogoevici - */ -public class KafkaTemplate implements KafkaOperations { - - private final ConnectionFactory connectionFactory; - - private FetchRequestToLeaderBrokerAddress fetchRequestToLeaderBrokerAddress = new FetchRequestToLeaderBrokerAddress(); - - public KafkaTemplate(ConnectionFactory connectionFactory) { - this.connectionFactory = connectionFactory; - } - - public ConnectionFactory getConnectionFactory() { - return connectionFactory; - } - - @Override - public Result receive(Iterable messageFetchRequests) { - FastList requestList = FastList.newList(messageFetchRequests); - MutableList distinctBrokerAddresses = - requestList.collect(fetchRequestToLeaderBrokerAddress).distinct(); - if (distinctBrokerAddresses.size() != 1) { - throw new IllegalArgumentException("All messages must be fetched from the same broker"); - } - return connectionFactory.connect(distinctBrokerAddresses.getFirst()) - .fetch(requestList.toTypedArray(FetchRequest.class)); - } - - @SuppressWarnings("serial") - private class FetchRequestToLeaderBrokerAddress implements Function { - - @Override - public BrokerAddress valueOf(FetchRequest fetchRequest) { - return connectionFactory.getLeader(fetchRequest.getPartition()); - } - - } - -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/MetadataCache.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/MetadataCache.java deleted file mode 100644 index 39fe907c7f..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/MetadataCache.java +++ /dev/null @@ -1,111 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.core; - -import java.util.Collection; -import java.util.Map; - -import com.gs.collections.api.block.function.Function; -import com.gs.collections.impl.map.mutable.UnifiedMap; -import com.gs.collections.impl.utility.Iterate; -import kafka.javaapi.PartitionMetadata; -import kafka.javaapi.TopicMetadata; - -/** - * Immutable store for the partition/broker mapping - * - * @author Marius Bogoevici - */ -class MetadataCache { - - public static final GetTopicNameFunction getTopicNameFunction = new GetTopicNameFunction(); - - public static final ToIndexedPartitionMetadataFunction toIndexedPartitionMetadataFunction = - new ToIndexedPartitionMetadataFunction(); - - private final Map> metadatasByTopic; - - public MetadataCache(Iterable topicMetadatas) { - UnifiedMap> topicData = UnifiedMap.newMap(); - for (TopicMetadata topicMetadata : topicMetadatas) { - Map partitionData = toPartitionMetadataMap(topicMetadata); - topicData.put(topicMetadata.topic(), partitionData); - } - this.metadatasByTopic = topicData.toImmutable().castToMap(); - } - - private MetadataCache(Map> metadatasByTopic) { - this.metadatasByTopic = metadatasByTopic; - } - - public MetadataCache merge(final Iterable topicMetadatas) { - UnifiedMap> unifiedMap = UnifiedMap.newMap(this.metadatasByTopic); - return new MetadataCache(Iterate.addToMap(topicMetadatas, getTopicNameFunction, - toIndexedPartitionMetadataFunction, unifiedMap).toImmutable().castToMap()); - } - - public Collection getPartitions(String topic) { - if (this.metadatasByTopic.containsKey(topic)) { - return this.metadatasByTopic.get(topic).keySet(); - } - else { - return null; - } - } - - public BrokerAddress getLeader(Partition partition) { - if (!this.metadatasByTopic.containsKey(partition.getTopic())) { - return null; - } - Map partitionMetadatasForTopic = this.metadatasByTopic.get(partition.getTopic()); - if (!partitionMetadatasForTopic.containsKey(partition)) { - return null; - } - return new BrokerAddress(partitionMetadatasForTopic.get(partition).leader()); - } - - - private static Map toPartitionMetadataMap(TopicMetadata topicMetadata) { - Map partitionData = UnifiedMap.newMap(); - for (PartitionMetadata partitionMetadata : topicMetadata.partitionsMetadata()) { - partitionData.put(new Partition(topicMetadata.topic(), partitionMetadata.partitionId()), partitionMetadata); - } - return partitionData; - } - - @SuppressWarnings("serial") - private static class GetTopicNameFunction implements Function { - - @Override - public String valueOf(TopicMetadata object) { - return object.topic(); - } - - } - - @SuppressWarnings("serial") - private static class ToIndexedPartitionMetadataFunction implements - Function> { - - @Override - public Map valueOf(TopicMetadata object) { - return toPartitionMetadataMap(object); - } - - } - -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/Partition.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/Partition.java deleted file mode 100644 index 819fdcaf55..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/Partition.java +++ /dev/null @@ -1,82 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.core; - -import org.springframework.util.Assert; - -/** - * A reference to a Kafka partition, with both topic and partition id - * - * @author Marius Bogoevici - */ -public class Partition { - - private String topic; - - private int id; - - public Partition(String topic, int id) { - Assert.hasText(topic, "Topic name cannot be empty"); - Assert.isTrue(id >= 0, "Partition id must be greater than or equal to 0"); - this.topic = topic; - this.id = id; - } - - public String getTopic() { - return topic; - } - - public void setTopic(String topic) { - this.topic = topic; - } - - public int getId() { - return id; - } - - public void setId(int id) { - this.id = id; - } - - @Override - public int hashCode() { - int result = topic.hashCode(); - result = 31 * result + id; - return result; - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - Partition partition = (Partition) o; - if (id != partition.id) { - return false; - } - return topic.equals(partition.topic); - } - - @Override - public String toString() { - return "Partition[" + "topic='" + topic + '\'' + ", id=" + id + ']'; - } - -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/PartitionNotFoundException.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/PartitionNotFoundException.java deleted file mode 100644 index 5c1f55c947..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/PartitionNotFoundException.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.core; - -/** - * @author Marius Bogoevici - */ -@SuppressWarnings("serial") -public class PartitionNotFoundException extends ConsumerException { - - public PartitionNotFoundException(Partition partition) { - super(String.format("Partition [%s,%d] has no leader or has not been found", - partition.getTopic(), partition.getId())); - } - -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/Result.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/Result.java deleted file mode 100644 index a2544a95f0..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/Result.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.core; - -import java.util.Collections; -import java.util.Map; - -/** - * The result of a Kafka SimpleConsumer operation - * - * @author Marius Bogoevici - */ -public class Result { - - private final Map results; - - private final Map errors; - - Result(Map results, Map errors) { - this.results = Collections.unmodifiableMap(results); - this.errors = Collections.unmodifiableMap(errors); - } - - public Map getResults() { - return results; - } - - public T getResult(Partition partition) throws IllegalArgumentException { - if (this.results.containsKey(partition)) { - return this.results.get(partition); - } - else { - throw new IllegalArgumentException(" No result received for " + partition.toString()); - } - } - - public Map getErrors() { - return errors; - } - - public short getError(Partition partition) throws IllegalArgumentException { - if (this.getErrors().containsKey(partition)) { - return this.getErrors().get(partition); - } - else { - throw new IllegalArgumentException("No error received for " + partition.toString()); - } - } - -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ResultBuilder.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ResultBuilder.java deleted file mode 100644 index fd2220d3ad..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ResultBuilder.java +++ /dev/null @@ -1,74 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.core; - -import java.util.HashMap; -import java.util.Map; - -/** - * Utility class for building a {@link Result} - * - * @author Marius Bogoevici - */ -class ResultBuilder { - - private Map result; - - private Map errors; - - public ResultBuilder() { - this.result = new HashMap(); - this.errors = new HashMap(); - } - - public KafkaPartitionResultHolder add(Partition Partition) { - return new KafkaPartitionResultHolder(Partition); - } - - public Result build() { - return new Result(result, errors); - } - - class KafkaPartitionResultHolder { - - private Partition partition; - - public KafkaPartitionResultHolder(Partition Partition) { - this.partition = Partition; - } - - public ResultBuilder withResult(T result) { - if (ResultBuilder.this.errors.containsKey(partition)) { - throw new IllegalArgumentException("A KafkaResult cannot contain both an error " + - "and a result for the same topic and partition"); - } - ResultBuilder.this.result.put(partition, result); - return ResultBuilder.this; - } - - public ResultBuilder withError(short error) { - if (ResultBuilder.this.result.containsKey(partition)) { - throw new IllegalArgumentException("A FetchResult cannot contain both an error " + - "and a MessageSet for the same topic and partition"); - } - ResultBuilder.this.errors.put(partition, error); - return ResultBuilder.this; - } - - } - -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/TopicNotFoundException.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/TopicNotFoundException.java deleted file mode 100644 index 2d99c0c2cf..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/TopicNotFoundException.java +++ /dev/null @@ -1,29 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package org.springframework.integration.kafka.core; - -/** - * @author Marius Bogoevici - */ -@SuppressWarnings("serial") -public class TopicNotFoundException extends ConsumerException { - - public TopicNotFoundException(String topicName) { - super(String.format("No topic named '%s' found", topicName)); - } -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ZookeeperConfiguration.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ZookeeperConfiguration.java deleted file mode 100644 index 543ac4e7d6..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ZookeeperConfiguration.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.core; - -import java.util.List; - -import org.I0Itec.zkclient.ZkClient; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import org.springframework.beans.factory.BeanInitializationException; -import org.springframework.integration.kafka.support.ZookeeperConnect; - -import com.gs.collections.api.block.function.Function; -import com.gs.collections.impl.list.mutable.FastList; - -import kafka.cluster.Broker; -import kafka.utils.ZKStringSerializer$; -import kafka.utils.ZkUtils$; -import scala.collection.JavaConversions; -import scala.collection.Seq; - -/** - * Kafka {@link Configuration} that uses a ZooKeeper connection for retrieving the list of seed brokers. - * - * @author Marius Bogoevici - */ -public class ZookeeperConfiguration extends AbstractConfiguration { - - private final static Log log = LogFactory.getLog(ZookeeperConfiguration.class); - - public static final BrokerToBrokerAddressFunction brokerToBrokerAddressFunction = new BrokerToBrokerAddressFunction(); - - private String zookeeperServers; - - private int sessionTimeout; - - private int connectionTimeout; - - public ZookeeperConfiguration(String zookeeperConnectionString) { - this(new ZookeeperConnect(zookeeperConnectionString)); - } - - public ZookeeperConfiguration(ZookeeperConnect zookeeperConnect) { - this.zookeeperServers = zookeeperConnect.getZkConnect(); - try { - this.sessionTimeout = Integer.parseInt(zookeeperConnect.getZkSessionTimeout()); - } - catch (NumberFormatException e) { - throw new BeanInitializationException("Cannot parse session timeout:", e); - } - try { - this.connectionTimeout = Integer.parseInt(zookeeperConnect.getZkConnectionTimeout()); - } - catch (NumberFormatException e) { - throw new BeanInitializationException("Cannot parse connection timeout:", e); - } - } - - @Override - protected List doGetBrokerAddresses() { - ZkClient zkClient = null; - try { - zkClient = new ZkClient(this.zookeeperServers, this.sessionTimeout, this.connectionTimeout, - ZKStringSerializer$.MODULE$); - Seq allBrokersInCluster = ZkUtils$.MODULE$.getAllBrokersInCluster(zkClient); - FastList brokers = FastList.newList(JavaConversions.asJavaCollection(allBrokersInCluster)); - return brokers.collect(brokerToBrokerAddressFunction); - } - finally { - if (zkClient != null) { - try { - zkClient.close(); - } - catch (Exception e) { - log.error("Cannot close Zookeeper client: ", e); - } - } - } - } - - @SuppressWarnings("serial") - private static class BrokerToBrokerAddressFunction implements Function { - - @Override - public BrokerAddress valueOf(Broker broker) { - return new BrokerAddress(broker.host(), broker.port()); - } - - } - -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ZookeeperConnectDefaults.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ZookeeperConnectDefaults.java deleted file mode 100644 index 03428d7577..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/ZookeeperConnectDefaults.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.kafka.core; - -/** - * - * @author Soby Chacko - * @since 0.5 - * - */ -public final class ZookeeperConnectDefaults { - public static final String ZK_CONNECT = "localhost:2181"; - public static final String ZK_CONNECTION_TIMEOUT = "6000"; - public static final String ZK_SESSION_TIMEOUT = "6000"; - public static final String ZK_SYNC_TIME = "2000"; - - private ZookeeperConnectDefaults() { - } -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/package-info.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/package-info.java deleted file mode 100644 index f404ec831c..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/core/package-info.java +++ /dev/null @@ -1,4 +0,0 @@ -/** - * Provides core classes of the Kafka module. - */ -package org.springframework.integration.kafka.core; diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/inbound/KafkaHighLevelConsumerMessageSource.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/inbound/KafkaHighLevelConsumerMessageSource.java deleted file mode 100644 index 348330f819..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/inbound/KafkaHighLevelConsumerMessageSource.java +++ /dev/null @@ -1,49 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.kafka.inbound; - -import java.util.List; -import java.util.Map; - -import org.springframework.integration.context.IntegrationObjectSupport; -import org.springframework.integration.core.MessageSource; -import org.springframework.messaging.Message; - -/** - * @author Soby Chacko - * @since 0.5 - * @deprecated since 1.3 in favor of {@link KafkaMessageDrivenChannelAdapter} - */ -@Deprecated -@SuppressWarnings("deprecation") -public class KafkaHighLevelConsumerMessageSource extends IntegrationObjectSupport implements MessageSource>>> { - - private final org.springframework.integration.kafka.support.KafkaConsumerContext kafkaConsumerContext; - - public KafkaHighLevelConsumerMessageSource(final org.springframework.integration.kafka.support.KafkaConsumerContext kafkaConsumerContext) { - this.kafkaConsumerContext = kafkaConsumerContext; - } - - @Override - public Message>>> receive() { - return kafkaConsumerContext.receive(); - } - - @Override - public String getComponentType() { - return "kafka:inbound-channel-adapter"; - } -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/inbound/KafkaMessageDrivenChannelAdapter.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/inbound/KafkaMessageDrivenChannelAdapter.java index e044e30887..2925d3a312 100644 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/inbound/KafkaMessageDrivenChannelAdapter.java +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/inbound/KafkaMessageDrivenChannelAdapter.java @@ -1,5 +1,5 @@ /* - * Copyright 2015 the original author or authors. + * Copyright 2015-2016 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -18,20 +18,19 @@ package org.springframework.integration.kafka.inbound; import java.util.Map; -import kafka.serializer.Decoder; -import kafka.serializer.DefaultDecoder; +import org.apache.kafka.clients.consumer.ConsumerRecord; import org.springframework.integration.context.OrderlyShutdownCapable; import org.springframework.integration.endpoint.MessageProducerSupport; -import org.springframework.integration.kafka.core.KafkaMessageMetadata; -import org.springframework.integration.kafka.listener.AbstractDecodingAcknowledgingMessageListener; -import org.springframework.integration.kafka.listener.AbstractDecodingMessageListener; -import org.springframework.integration.kafka.listener.Acknowledgment; -import org.springframework.integration.kafka.listener.KafkaMessageListenerContainer; -import org.springframework.integration.kafka.support.KafkaHeaders; import org.springframework.integration.support.DefaultMessageBuilderFactory; import org.springframework.integration.support.MessageBuilderFactory; import org.springframework.integration.support.MutableMessageBuilderFactory; +import org.springframework.kafka.listener.AbstractMessageListenerContainer; +import org.springframework.kafka.listener.AbstractMessageListenerContainer.AckMode; +import org.springframework.kafka.listener.AcknowledgingMessageListener; +import org.springframework.kafka.listener.MessageListener; +import org.springframework.kafka.support.Acknowledgment; +import org.springframework.kafka.support.KafkaHeaders; import org.springframework.messaging.Message; import org.springframework.messaging.MessageHeaders; import org.springframework.messaging.support.MessageBuilder; @@ -39,14 +38,13 @@ import org.springframework.util.Assert; /** * @author Marius Bogoevici + * @author Gary Russell + * + * TODO: Use the MessagingMessageConverter from spring-kafka */ -public class KafkaMessageDrivenChannelAdapter extends MessageProducerSupport implements OrderlyShutdownCapable { +public class KafkaMessageDrivenChannelAdapter extends MessageProducerSupport implements OrderlyShutdownCapable { - private final KafkaMessageListenerContainer messageListenerContainer; - - private Decoder keyDecoder = new DefaultDecoder(null); - - private Decoder payloadDecoder = new DefaultDecoder(null); + private final AbstractMessageListenerContainer messageListenerContainer; private boolean generateMessageId = false; @@ -54,40 +52,19 @@ public class KafkaMessageDrivenChannelAdapter extends MessageProducerSupport imp private boolean useMessageBuilderFactory = false; - private boolean autoCommitOffset = true; - - public KafkaMessageDrivenChannelAdapter(KafkaMessageListenerContainer messageListenerContainer) { - Assert.notNull(messageListenerContainer); - Assert.isNull(messageListenerContainer.getMessageListener()); + public KafkaMessageDrivenChannelAdapter(AbstractMessageListenerContainer messageListenerContainer) { + Assert.notNull(messageListenerContainer, "messageListenerContainer is required"); + Assert.isNull(messageListenerContainer.getMessageListener(), "Container must not already have a listener"); this.messageListenerContainer = messageListenerContainer; this.messageListenerContainer.setAutoStartup(false); } - public void setKeyDecoder(Decoder keyDecoder) { - this.keyDecoder = keyDecoder; - } - - public void setPayloadDecoder(Decoder payloadDecoder) { - this.payloadDecoder = payloadDecoder; - } - /** - * Automatically commit the offsets when 'true'. When 'false', the - * adapter inserts a 'kafka_acknowledgment` header allowing the user to manually - * commit the offset using the {@link Acknowledgment#acknowledge()} method. - * Default 'true'. - * @param autoCommitOffset false to not auto-commit (default true). - */ - public void setAutoCommitOffset(boolean autoCommitOffset) { - this.autoCommitOffset = autoCommitOffset; - } - - /** - * Generate {@link Message} {@code ids} for produced messages. - * If set to {@code false}, will try to use a default value. By default set to {@code false}. - * Note that this option is only guaranteed to work when - * {@link #setUseMessageBuilderFactory(boolean) useMessageBuilderFactory} is false (default). - * If the latter is set to {@code true}, then some {@link MessageBuilderFactory} implementations such as + * Generate {@link Message} {@code ids} for produced messages. If set to {@code false} + * , will try to use a default value. By default set to {@code false}. Note that this + * option is only guaranteed to work when {@link #setUseMessageBuilderFactory(boolean) + * useMessageBuilderFactory} is false (default). If the latter is set to {@code true}, + * then some {@link MessageBuilderFactory} implementations such as * {@link DefaultMessageBuilderFactory} may ignore it. * @param generateMessageId true if a message id should be generated * @since 1.1 @@ -97,11 +74,11 @@ public class KafkaMessageDrivenChannelAdapter extends MessageProducerSupport imp } /** - * Generate {@code timestamp} for produced messages. If set to {@code false}, -1 is used instead. - * By default set to {@code false}. - * Note that this option is only guaranteed to work when - * {@link #setUseMessageBuilderFactory(boolean) useMessageBuilderFactory} is false (default). - * If the latter is set to {@code true}, then some {@link MessageBuilderFactory} implementations such as + * Generate {@code timestamp} for produced messages. If set to {@code false}, -1 is + * used instead. By default set to {@code false}. Note that this option is only + * guaranteed to work when {@link #setUseMessageBuilderFactory(boolean) + * useMessageBuilderFactory} is false (default). If the latter is set to {@code true}, + * then some {@link MessageBuilderFactory} implementations such as * {@link DefaultMessageBuilderFactory} may ignore it. * @param generateTimestamp true if a timestamp should be generated * @since 1.1 @@ -111,9 +88,10 @@ public class KafkaMessageDrivenChannelAdapter extends MessageProducerSupport imp } /** - * Use the {@link MessageBuilderFactory} returned by {@link #getMessageBuilderFactory()} to create messages. - * @param useMessageBuilderFactory true if the {@link MessageBuilderFactory} returned by - * {@link #getMessageBuilderFactory()} should be used. + * Use the {@link MessageBuilderFactory} returned by + * {@link #getMessageBuilderFactory()} to create messages. + * @param useMessageBuilderFactory true if the {@link MessageBuilderFactory} returned + * by {@link #getMessageBuilderFactory()} should be used. * @since 1.1 */ public void setUseMessageBuilderFactory(boolean useMessageBuilderFactory) { @@ -122,9 +100,10 @@ public class KafkaMessageDrivenChannelAdapter extends MessageProducerSupport imp @Override protected void onInit() { - this.messageListenerContainer.setMessageListener(autoCommitOffset ? - new AutoAcknowledgingChannelForwardingMessageListener() - : new AcknowledgingChannelForwardingMessageListener()); + this.messageListenerContainer.setMessageListener( + !AckMode.MANUAL.equals(this.messageListenerContainer.getAckMode()) + ? new AutoAcknowledgingChannelForwardingMessageListener() + : new AcknowledgingChannelForwardingMessageListener()); if (!this.generateMessageId && !this.generateTimestamp && (getMessageBuilderFactory() instanceof DefaultMessageBuilderFactory)) { setMessageBuilderFactory(new MutableMessageBuilderFactory()); @@ -158,61 +137,46 @@ public class KafkaMessageDrivenChannelAdapter extends MessageProducerSupport imp return getPhase(); } - @SuppressWarnings("rawtypes") - private class AutoAcknowledgingChannelForwardingMessageListener extends AbstractDecodingMessageListener { - - @SuppressWarnings("unchecked") - public AutoAcknowledgingChannelForwardingMessageListener() { - super(keyDecoder, payloadDecoder); - } + private class AutoAcknowledgingChannelForwardingMessageListener implements MessageListener { @Override - public void doOnMessage(Object key, Object payload, KafkaMessageMetadata metadata) { - sendMessage(toMessage(key, payload, metadata, null)); + public void onMessage(ConsumerRecord record) { + sendMessage(toMessage(record, null)); } } - @SuppressWarnings("rawtypes") - private class AcknowledgingChannelForwardingMessageListener extends AbstractDecodingAcknowledgingMessageListener { - - @SuppressWarnings("unchecked") - public AcknowledgingChannelForwardingMessageListener() { - super(keyDecoder, payloadDecoder); - } + private class AcknowledgingChannelForwardingMessageListener implements AcknowledgingMessageListener { @Override - public void doOnMessage(Object key, Object payload, KafkaMessageMetadata metadata, - Acknowledgment acknowledgment) { - - sendMessage(toMessage(key, payload, metadata, acknowledgment)); + public void onMessage(ConsumerRecord record, Acknowledgment acknowledgment) { + sendMessage(toMessage(record, acknowledgment)); } + } - private Message toMessage(Object key, Object payload, KafkaMessageMetadata metadata, - Acknowledgment acknowledgment) { + private Message toMessage(ConsumerRecord record, Acknowledgment acknowledgment) { KafkaMessageHeaders kafkaMessageHeaders = new KafkaMessageHeaders(generateMessageId, generateTimestamp); Map rawHeaders = kafkaMessageHeaders.getRawHeaders(); - rawHeaders.put(KafkaHeaders.MESSAGE_KEY, key); - rawHeaders.put(KafkaHeaders.TOPIC, metadata.getPartition().getTopic()); - rawHeaders.put(KafkaHeaders.PARTITION_ID, metadata.getPartition().getId()); - rawHeaders.put(KafkaHeaders.OFFSET, metadata.getOffset()); - rawHeaders.put(KafkaHeaders.NEXT_OFFSET, metadata.getNextOffset()); + rawHeaders.put(KafkaHeaders.MESSAGE_KEY, record.key()); + rawHeaders.put(KafkaHeaders.TOPIC, record.topic()); + rawHeaders.put(KafkaHeaders.PARTITION_ID, record.partition()); + rawHeaders.put(KafkaHeaders.OFFSET, record.offset()); - if (!this.autoCommitOffset) { + if (acknowledgment != null) { rawHeaders.put(KafkaHeaders.ACKNOWLEDGMENT, acknowledgment); } if (this.useMessageBuilderFactory) { return getMessageBuilderFactory() - .withPayload(payload) + .withPayload(record.value()) .copyHeaders(kafkaMessageHeaders) .build(); } else { - return MessageBuilder.createMessage(payload, kafkaMessageHeaders); + return MessageBuilder.createMessage(record.value(), kafkaMessageHeaders); } } @@ -229,4 +193,5 @@ public class KafkaMessageDrivenChannelAdapter extends MessageProducerSupport imp } } + } diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/AbstractDecodingAcknowledgingMessageListener.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/AbstractDecodingAcknowledgingMessageListener.java deleted file mode 100644 index 8355f0fecd..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/AbstractDecodingAcknowledgingMessageListener.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.listener; - - -import org.springframework.integration.kafka.core.KafkaMessage; -import org.springframework.integration.kafka.core.KafkaMessageMetadata; -import org.springframework.integration.kafka.util.MessageUtils; - -import kafka.serializer.Decoder; - -/** - * Base {@link AcknowledgingMessageListener} implementation that decodes the key and the - * payload using the supplied {@link Decoder}s. - * - * Users of this class must extend it and implement {@code doOnMessage} and must supply - * {@link Decoder} implementations for both the key and the payload. - * - * @author Marius Bogoevici - * @since 1.0.1 - */ -public abstract class AbstractDecodingAcknowledgingMessageListener implements AcknowledgingMessageListener { - - private final Decoder keyDecoder; - - private final Decoder

payloadDecoder; - - public AbstractDecodingAcknowledgingMessageListener(Decoder keyDecoder, Decoder

payloadDecoder) { - this.keyDecoder = keyDecoder; - this.payloadDecoder = payloadDecoder; - } - - @Override - public final void onMessage(KafkaMessage message, Acknowledgment acknowledgment) { - this.doOnMessage(MessageUtils.decodeKey(message, keyDecoder), - MessageUtils.decodePayload(message, payloadDecoder), message.getMetadata(), acknowledgment); - } - - /** - * Process the decoded message - * @param key the message key - * @param payload the message body - * @param metadata the KafkaMessageMetadata - * @param acknowledgment the acknowledgment handle - */ - public abstract void doOnMessage(K key, P payload, KafkaMessageMetadata metadata, Acknowledgment acknowledgment); - -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/AbstractDecodingMessageListener.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/AbstractDecodingMessageListener.java deleted file mode 100644 index 9db10afa38..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/AbstractDecodingMessageListener.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.listener; - - -import kafka.serializer.Decoder; - -import org.springframework.beans.factory.InitializingBean; -import org.springframework.integration.kafka.core.KafkaMessage; -import org.springframework.integration.kafka.core.KafkaMessageMetadata; -import org.springframework.integration.kafka.util.MessageUtils; -import org.springframework.util.Assert; - -/** - * Base {@link MessageListener} implementation that decodes the key and the payload using the supplied - * {@link Decoder}s. - * - * Users of this class must extend it and implement {@code doOnMessage} and must supply {@link Decoder} - * implementations for both the key and the payload. - * - * @author Marius Bogoevici - */ -public abstract class AbstractDecodingMessageListener implements MessageListener { - - private Decoder keyDecoder; - - private Decoder

payloadDecoder; - - public AbstractDecodingMessageListener(Decoder keyDecoder, Decoder

payloadDecoder) { - this.keyDecoder = keyDecoder; - this.payloadDecoder = payloadDecoder; - } - - @Override - public final void onMessage(KafkaMessage message) { - this.doOnMessage(MessageUtils.decodeKey(message, keyDecoder), - MessageUtils.decodePayload(message, payloadDecoder), message.getMetadata()); - } - - /** - * Process the decoded message - * @param key the message key - * @param payload the message body - * @param metadata the KafkaMessageMetadata - */ - public abstract void doOnMessage(K key, P payload, KafkaMessageMetadata metadata); - -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/AbstractOffsetManager.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/AbstractOffsetManager.java deleted file mode 100644 index f38a7ba7a6..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/AbstractOffsetManager.java +++ /dev/null @@ -1,176 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.listener; - -import java.io.IOException; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -import kafka.common.ErrorMapping; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import org.springframework.beans.factory.DisposableBean; -import org.springframework.integration.kafka.core.BrokerAddress; -import org.springframework.integration.kafka.core.Connection; -import org.springframework.integration.kafka.core.ConnectionFactory; -import org.springframework.integration.kafka.core.ConsumerException; -import org.springframework.integration.kafka.core.KafkaConsumerDefaults; -import org.springframework.integration.kafka.core.Partition; -import org.springframework.integration.kafka.core.PartitionNotFoundException; -import org.springframework.integration.kafka.core.Result; -import org.springframework.util.Assert; - -/** - * Base implementation for {@link OffsetManager}. Subclasses may customize functionality as necessary. - * - * @author Marius Bogoevici - */ -public abstract class AbstractOffsetManager implements OffsetManager, DisposableBean { - - protected final Log log = LogFactory.getLog(this.getClass()); - - protected String consumerId = KafkaConsumerDefaults.GROUP_ID; - - protected long referenceTimestamp = KafkaConsumerDefaults.DEFAULT_OFFSET_RESET; - - protected ConnectionFactory connectionFactory; - - protected Map initialOffsets; - - protected Map highestUpdatedOffsets = new ConcurrentHashMap(); - - public AbstractOffsetManager(ConnectionFactory connectionFactory) { - this(connectionFactory, new HashMap()); - } - - public AbstractOffsetManager(ConnectionFactory connectionFactory, Map initialOffsets) { - Assert.notNull(connectionFactory, "A 'connectionFactory' can't be null"); - Assert.notNull(initialOffsets, "An initialOffsets can't be null"); - this.connectionFactory = connectionFactory; - this.initialOffsets = new HashMap(initialOffsets); - } - - public String getConsumerId() { - return this.consumerId; - } - - /** - * The identifier of a consumer of Kafka messages. Allows to manage offsets separately by consumer - * - * @param consumerId the consumer ID - */ - public void setConsumerId(String consumerId) { - this.consumerId = consumerId; - } - - /** - * A timestamp to be used for resetting initial offsets - * - * @param referenceTimestamp the reset timestamp for initial offsets - */ - public void setReferenceTimestamp(long referenceTimestamp) { - this.referenceTimestamp = referenceTimestamp; - } - - @Override - public void destroy() throws Exception { - try { - this.flush(); - } - catch (IOException e) { - log.error("Error while flushing the OffsetManager", e); - } - try { - this.close(); - } - catch (IOException e) { - log.error("Error while closing the OffsetManager", e); - } - } - - /** - * @see OffsetManager#updateOffset(Partition, long) - */ - @Override - public synchronized final void updateOffset(Partition partition, long offset) { - Long highestUpdatedOffset = this.highestUpdatedOffsets.get(partition); - if (highestUpdatedOffset == null || highestUpdatedOffset < offset) { - highestUpdatedOffsets.put(partition, offset); - doUpdateOffset(partition, offset); - } - } - - /** - * @see OffsetManager#getOffset(Partition) - */ - @Override - public synchronized final long getOffset(Partition partition) { - Long storedOffset = doGetOffset(partition); - if (storedOffset == null) { - if (this.initialOffsets.containsKey(partition)) { - return this.initialOffsets.get(partition); - } - else { - BrokerAddress leader = this.connectionFactory.getLeader(partition); - if (leader == null) { - throw new PartitionNotFoundException(partition); - } - Connection connection = this.connectionFactory.connect(leader); - Result offsetResult = connection.fetchInitialOffset(this.referenceTimestamp, partition); - if (offsetResult.getErrors().size() > 0) { - short errorCode = offsetResult.getError(partition); - if (log.isWarnEnabled()) { - log.warn("Error code while retrieving offset for partition " + partition.toString() + " : " + errorCode); - } - throw new ConsumerException(ErrorMapping.exceptionFor(errorCode)); - } - if (!offsetResult.getResults().containsKey(partition)) { - throw new IllegalStateException("Result does not contain an expected value"); - } - return offsetResult.getResult(partition); - } - } - else { - return storedOffset; - } - } - - @Override - public synchronized void resetOffsets(Collection partitionsToReset) { - // any information about those offsets is invalid and can be ignored - for (Partition partition : partitionsToReset) { - doRemoveOffset(partition); - this.initialOffsets.remove(partition); - this.highestUpdatedOffsets.remove(partition); - } - } - - @Override - public synchronized void deleteOffset(Partition partition) { - doRemoveOffset(partition); - } - - protected abstract void doUpdateOffset(Partition partition, long offset); - - protected abstract void doRemoveOffset(Partition partition); - - protected abstract Long doGetOffset(Partition partition); - -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/AcknowledgingMessageListener.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/AcknowledgingMessageListener.java deleted file mode 100644 index 0ec05e0f15..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/AcknowledgingMessageListener.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.listener; - -import org.springframework.integration.kafka.core.KafkaMessage; - -/** - * Listener for handling incoming Kafka messages, propagating an acknowledgment handle that recipients - * can invoke when the message has been processed. - * - * @author Marius Bogoevici - * @since 1.0.1 - */ -public interface AcknowledgingMessageListener { - - /** - * Executes when a Kafka message is received - * - * @param message the Kafka message to be processed - * @param acknowledgment a handle for acknowledging the message processing - */ - void onMessage(KafkaMessage message, Acknowledgment acknowledgment); - -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/Acknowledgment.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/Acknowledgment.java deleted file mode 100644 index 1dda71edc6..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/Acknowledgment.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.listener; - -import org.springframework.integration.kafka.core.KafkaMessage; - -/** - * Handle for acknowledging the processing of a {@link KafkaMessage}. Recipients can store the reference in - * asynchronous scenarios, but the internal state should be assumed transient (i.e. it cannot be serialized - * and deserialized later) - * - * @author Marius Bogoevici - * @since 1.0.1 - */ -public interface Acknowledgment { - - /** - * Invoked when the message for which the acknowledgment has been created has been processed. - * Calling this method implies that all the previous messages in the partition have been processed already. - */ - void acknowledge(); - -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/ConcurrentMessageListenerDispatcher.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/ConcurrentMessageListenerDispatcher.java deleted file mode 100644 index 68f789aad1..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/ConcurrentMessageListenerDispatcher.java +++ /dev/null @@ -1,167 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.listener; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.concurrent.Executor; - -import com.gs.collections.api.block.procedure.Procedure; -import com.gs.collections.api.block.procedure.Procedure2; -import com.gs.collections.api.map.MutableMap; -import com.gs.collections.impl.factory.Maps; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import org.springframework.integration.kafka.core.KafkaMessage; -import org.springframework.integration.kafka.core.Partition; -import org.springframework.util.Assert; - -/** - * Dispatches {@link KafkaMessage}s to a {@link MessageListener}. Messages may be - * processed concurrently, according to the {@code concurrency} settings, but messages - * from the same partition are being processed in their original order. - * - * @author Marius Bogoevici - * @author Artem Bilan - */ -class ConcurrentMessageListenerDispatcher { - - private static final Log log = LogFactory.getLog(ConcurrentMessageListenerDispatcher.class); - - private static final StartDelegateProcedure startDelegateProcedure = new StartDelegateProcedure(); - - private static final StopDelegateProcedure stopDelegateProcedure = new StopDelegateProcedure(); - - private final Object lifecycleMonitor = new Object(); - - private final Collection partitions; - - private final int consumers; - - private final Object delegateListener; - - private final ErrorHandler errorHandler; - - private final OffsetManager offsetManager; - - private final int queueSize; - - private final Executor taskExecutor; - - private volatile boolean running; - - private MutableMap delegates; - - private boolean autoCommitOnError; - - public ConcurrentMessageListenerDispatcher(Object delegateListener, ErrorHandler errorHandler, - Collection partitions, OffsetManager offsetManager, int consumers, int queueSize, - Executor taskExecutor, boolean autoCommitOnError) { - Assert.isTrue( - delegateListener instanceof MessageListener || delegateListener instanceof AcknowledgingMessageListener, - "Either a " + MessageListener.class.getName() + " or a " + AcknowledgingMessageListener.class.getName() - + " must be provided"); - Assert.notEmpty(partitions, "A set of partitions must be provided"); - Assert.isTrue(consumers <= partitions.size(), - "The number of consumers must be smaller or equal to the number of partitions"); - Assert.notNull(delegateListener, "A delegate must be provided"); - this.delegateListener = delegateListener; - this.errorHandler = errorHandler; - this.partitions = partitions; - this.offsetManager = offsetManager; - this.consumers = Math.min(partitions.size(), consumers); - this.queueSize = queueSize; - this.taskExecutor = taskExecutor; - this.autoCommitOnError = autoCommitOnError; - } - - public void start() { - synchronized (lifecycleMonitor) { - if (!this.running) { - initializeAndStartDispatching(); - this.running = true; - } - } - } - - public void stop(int stopTimeout) { - synchronized (lifecycleMonitor) { - if (this.running) { - this.running = false; - delegates.flip().keyBag().toSet().forEachWith(stopDelegateProcedure, stopTimeout); - } - this.delegates = null; - } - } - - public void dispatch(KafkaMessage message) { - if (this.running) { - delegates.get(message.getMetadata().getPartition()).enqueue(message); - } - } - - private void initializeAndStartDispatching() { - // allocate delegate instances index them - List delegateList = new ArrayList<>(consumers); - for (int i = 0; i < consumers; i++) { - QueueingMessageListenerInvoker queueingMessageListenerInvoker = new QueueingMessageListenerInvoker( - queueSize, offsetManager, delegateListener, errorHandler, taskExecutor, autoCommitOnError); - delegateList.add(queueingMessageListenerInvoker); - } - - // evenly distribute partitions across delegates - int i = 0; - delegates = Maps.mutable.of(); - for (Partition partition : partitions) { - delegates.put(partition, delegateList.get((i++) % consumers)); - } - - // start dispatchers - delegates.flip().keyBag().toSet().forEach(startDelegateProcedure); - } - - @SuppressWarnings("serial") - private static class StopDelegateProcedure implements Procedure2 { - - @Override - public void value(QueueingMessageListenerInvoker delegate, Integer stopTimeout) { - try { - delegate.stop(stopTimeout); - } - catch (Exception e) { - // ignore the exception, but log it - if (log.isInfoEnabled()) { - log.info("Exception thrown while stopping dispatcher:", e); - } - } - } - - } - - @SuppressWarnings("serial") - private static class StartDelegateProcedure implements Procedure { - - @Override - public void value(QueueingMessageListenerInvoker delegate) { - delegate.start(); - } - - } - -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/DefaultAcknowledgment.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/DefaultAcknowledgment.java deleted file mode 100644 index a4c16f930e..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/DefaultAcknowledgment.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.listener; - -import org.springframework.integration.kafka.core.KafkaMessage; -import org.springframework.integration.kafka.core.Partition; - -/** - * Default implementation for an {@link Acknowledgment} that defers to an underlying - * {@link OffsetManager}. - * - * @author Marius Bogoevici - * @since 1.0.1 - */ -public class DefaultAcknowledgment implements Acknowledgment { - - private final OffsetManager offsetManager; - - private final Partition partition; - - private final Long offset; - - public DefaultAcknowledgment(OffsetManager offsetManager, Partition partition, Long offset) { - this.offsetManager = offsetManager; - this.partition = partition; - this.offset = offset; - } - - public DefaultAcknowledgment(OffsetManager offsetManager, KafkaMessage message) { - this(offsetManager, message.getMetadata().getPartition(), message.getMetadata().getNextOffset()); - } - - @Override - public void acknowledge() { - offsetManager.updateOffset(partition, offset); - } - -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/ErrorHandler.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/ErrorHandler.java deleted file mode 100644 index 259389375b..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/ErrorHandler.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package org.springframework.integration.kafka.listener; - -import org.springframework.integration.kafka.core.KafkaMessage; - -/** - * Handles errors thrown during the execution of a {@link MessageListener} - * - * @author Marius Bogoevici - */ -public interface ErrorHandler { - - void handle(Exception thrownException, KafkaMessage message); - -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/KafkaMessageListenerContainer.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/KafkaMessageListenerContainer.java deleted file mode 100644 index b8edaa7aab..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/KafkaMessageListenerContainer.java +++ /dev/null @@ -1,655 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.listener; - -import java.io.IOException; -import java.util.Arrays; -import java.util.Collection; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.Executor; -import java.util.concurrent.Executors; - -import com.gs.collections.api.RichIterable; -import com.gs.collections.api.block.function.Function; -import com.gs.collections.api.block.predicate.Predicate; -import com.gs.collections.api.list.ImmutableList; -import com.gs.collections.api.list.MutableList; -import com.gs.collections.api.multimap.list.ImmutableListMultimap; -import com.gs.collections.api.multimap.set.MutableSetMultimap; -import com.gs.collections.api.partition.PartitionIterable; -import com.gs.collections.api.set.MutableSet; -import com.gs.collections.api.tuple.Pair; -import com.gs.collections.impl.block.factory.Functions; -import com.gs.collections.impl.block.function.checked.CheckedFunction; -import com.gs.collections.impl.factory.Lists; -import com.gs.collections.impl.factory.Sets; -import com.gs.collections.impl.list.mutable.FastList; -import com.gs.collections.impl.map.mutable.UnifiedMap; -import com.gs.collections.impl.utility.ArrayIterate; -import com.gs.collections.impl.utility.Iterate; -import kafka.common.ErrorMapping; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import org.springframework.context.SmartLifecycle; -import org.springframework.core.task.SimpleAsyncTaskExecutor; -import org.springframework.integration.kafka.core.BrokerAddress; -import org.springframework.integration.kafka.core.ConnectionFactory; -import org.springframework.integration.kafka.core.ConsumerException; -import org.springframework.integration.kafka.core.FetchRequest; -import org.springframework.integration.kafka.core.KafkaConsumerDefaults; -import org.springframework.integration.kafka.core.KafkaMessage; -import org.springframework.integration.kafka.core.KafkaMessageBatch; -import org.springframework.integration.kafka.core.KafkaTemplate; -import org.springframework.integration.kafka.core.Partition; -import org.springframework.integration.kafka.core.Result; -import org.springframework.scheduling.SchedulingAwareRunnable; -import org.springframework.util.Assert; - -/** - * @author Marius Bogoevici - */ -public class KafkaMessageListenerContainer implements SmartLifecycle { - - public static final int DEFAULT_WAIT_FOR_LEADER_REFRESH_RETRY = 5000; - - private static final int DEFAULT_STOP_TIMEOUT = 1000; - - private static final Log log = LogFactory.getLog(KafkaMessageListenerContainer.class); - - private final GetOffsetForPartitionFunction getOffset = new GetOffsetForPartitionFunction(); - - private final PartitionToLeaderFunction getLeader = new PartitionToLeaderFunction(); - - private final Function passThru = Functions.getPassThru(); - - private final Object lifecycleMonitor = new Object(); - - private final KafkaTemplate kafkaTemplate; - - private final String[] topics; - - private Partition[] partitions; - - public boolean autoStartup = true; - - private Executor fetchTaskExecutor; - - private Executor adminTaskExecutor; - - private Executor dispatcherTaskExecutor; - - private int concurrency = 1; - - private volatile boolean running = false; - - private int maxFetch = KafkaConsumerDefaults.FETCH_SIZE_INT; - - private int queueSize = 1024; - - private int stopTimeout = DEFAULT_STOP_TIMEOUT; - - private Object messageListener; - - private ErrorHandler errorHandler = new LoggingErrorHandler(); - - private volatile OffsetManager offsetManager; - - private ConcurrentMap fetchOffsets; - - private ConcurrentMessageListenerDispatcher messageDispatcher; - - private final ConcurrentMap fetchTasksByBroker = new ConcurrentHashMap<>(); - - private boolean autoCommitOnError = false; - - public KafkaMessageListenerContainer(ConnectionFactory connectionFactory, Partition... partitions) { - Assert.notNull(connectionFactory, "A connection factory must be supplied"); - Assert.notEmpty(partitions, "A list of partitions must be provided"); - Assert.noNullElements(partitions, "The list of partitions cannot contain null elements"); - this.kafkaTemplate = new KafkaTemplate(connectionFactory); - this.partitions = partitions; - this.topics = null; - } - - public KafkaMessageListenerContainer(final ConnectionFactory connectionFactory, String... topics) { - Assert.notNull(connectionFactory, "A connection factory must be supplied"); - Assert.notNull(topics, "A list of topics must be provided"); - Assert.noNullElements(topics, "The list of topics cannot contain null elements"); - this.kafkaTemplate = new KafkaTemplate(connectionFactory); - this.topics = topics; - } - - public OffsetManager getOffsetManager() { - return offsetManager; - } - - public void setOffsetManager(OffsetManager offsetManager) { - this.offsetManager = offsetManager; - } - - public Object getMessageListener() { - return messageListener; - } - - public void setMessageListener(Object messageListener) { - Assert.isTrue( - messageListener instanceof MessageListener || messageListener instanceof AcknowledgingMessageListener, - "Either a " + MessageListener.class.getName() + " or a " + AcknowledgingMessageListener.class.getName() - + " must be provided"); - this.messageListener = messageListener; - } - - public ErrorHandler getErrorHandler() { - return errorHandler; - } - - public void setErrorHandler(ErrorHandler errorHandler) { - this.errorHandler = errorHandler; - } - - public int getConcurrency() { - return concurrency; - } - - /** - * The maximum number of concurrent {@link MessageListener}s running. Messages from - * within the same partition will be processed sequentially. - * @param concurrency the concurrency maximum number - */ - public void setConcurrency(int concurrency) { - this.concurrency = concurrency; - } - - /** - * The timeout for waiting for each concurrent {@link MessageListener} to finish on - * stopping. - * @param stopTimeout timeout in milliseconds - * @since 1.1 - */ - public void setStopTimeout(int stopTimeout) { - this.stopTimeout = stopTimeout; - } - - public int getStopTimeout() { - return stopTimeout; - } - - public Executor getFetchTaskExecutor() { - return fetchTaskExecutor; - } - - /** - * The task executor for fetch operations. - * @param fetchTaskExecutor the Executor for fetch operations - */ - public void setFetchTaskExecutor(Executor fetchTaskExecutor) { - this.fetchTaskExecutor = fetchTaskExecutor; - } - - public Executor getAdminTaskExecutor() { - return adminTaskExecutor; - } - - /** - * The task executor for leader, offset, and partition reassignment updates. - * @param adminTaskExecutor the task executor for leader, offset and partition reassignment updates - */ - public void setAdminTaskExecutor(Executor adminTaskExecutor) { - this.adminTaskExecutor = adminTaskExecutor; - } - - /** - * The task executor for invoking the MessageListener - * @param dispatcherTaskExecutor the task executor for invoking the MessageListener - */ - public void setDispatcherTaskExecutor(Executor dispatcherTaskExecutor) { - this.dispatcherTaskExecutor = dispatcherTaskExecutor; - } - - /** - * @return the maximum amount of data (in bytes) that pollers will fetch in one round - */ - public int getMaxFetch() { - return maxFetch; - } - - public int getQueueSize() { - return queueSize; - } - - /** - * The maximum number of messages that are buffered by each concurrent - * {@link MessageListener} runner. Increasing the value may increase throughput, but - * also increases the memory consumption. Must be a positive number and a power of 2. - * @param queueSize the queue size - */ - public void setQueueSize(int queueSize) { - Assert.isTrue(queueSize > 0 && Integer.bitCount(queueSize) == 1, - "'queueSize' must be a positive number and a power of 2"); - this.queueSize = queueSize; - } - - public void setMaxFetch(int maxFetch) { - this.maxFetch = maxFetch; - } - - /** - * Whether offsets should be auto acknowledged even when exceptions are thrown during processing. This setting - * is effective only in auto acknowledged mode. When set to true, all received messages will be acknowledged, - * and when set to false only the offset of the last successfully processed message is persisted, even if the - * component will try to continue processing incoming messages. In the latter case, it is possible that - * a successful message will commit an offset after a series of failures, so the component should rely on - * the `errorHandler` to capture failures. - * @param autoCommitOnError false if offsets should be committed only for successful messages - * @since 1.3 - */ - public void setAutoCommitOnError(boolean autoCommitOnError) { - this.autoCommitOnError = autoCommitOnError; - } - - public boolean isAutoCommitOnError() { - return autoCommitOnError; - } - - @Override - public boolean isAutoStartup() { - return autoStartup; - } - - public void setAutoStartup(boolean autoStartup) { - this.autoStartup = autoStartup; - } - - @Override - public void stop(Runnable callback) { - synchronized (lifecycleMonitor) { - if (running) { - this.running = false; - try { - this.offsetManager.flush(); - } - catch (IOException e) { - log.error("Error while flushing:", e); - } - this.messageDispatcher.stop(stopTimeout); - } - } - if (callback != null) { - callback.run(); - } - } - - @Override - public void start() { - synchronized (lifecycleMonitor) { - if (!running) { - if (partitions == null) { - partitions = getPartitionsForTopics(kafkaTemplate.getConnectionFactory(), topics); - } - this.running = true; - if (this.offsetManager == null) { - this.offsetManager = new MetadataStoreOffsetManager(kafkaTemplate.getConnectionFactory()); - } - // initialize the fetch offset table - defer to OffsetManager for retrieving them - ImmutableList partitionsAsList = Lists.immutable.with(partitions); - this.fetchOffsets = new ConcurrentHashMap(partitionsAsList.toMap(passThru, getOffset)); - this.messageDispatcher = new ConcurrentMessageListenerDispatcher(messageListener, errorHandler, - Arrays.asList(partitions), offsetManager, concurrency, queueSize, dispatcherTaskExecutor, - autoCommitOnError); - this.messageDispatcher.start(); - fetchTasksByBroker.clear(); - ImmutableListMultimap partitionsByLeader = partitionsAsList - .groupBy(getLeader); - if (fetchTaskExecutor == null) { - fetchTaskExecutor = new SimpleAsyncTaskExecutor("kafka-fetch-"); - } - if (adminTaskExecutor == null) { - adminTaskExecutor = Executors.newSingleThreadExecutor(); - } - for (Pair> entry : partitionsByLeader.keyMultiValuePairsView()) { - FetchTask fetchTask = new FetchTask(entry.getOne(), entry.getTwo()); - fetchTaskExecutor.execute(fetchTask); - fetchTasksByBroker.put(entry.getOne(), fetchTask); - } - } - } - } - - @Override - public void stop() { - this.stop(null); - } - - @Override - public boolean isRunning() { - return this.running; - } - - @Override - public int getPhase() { - return 0; - } - - private static Partition[] getPartitionsForTopics(final ConnectionFactory connectionFactory, String[] topics) { - MutableList partitionList = - ArrayIterate.flatCollect(topics, new GetPartitionsForTopic(connectionFactory)); - return partitionList.toArray(new Partition[partitionList.size()]); - } - - /** - * Fetches data from Kafka for a group of partitions, located on the same broker. - */ - public class FetchTask implements SchedulingAwareRunnable { - - private final BrokerAddress brokerAddress; - - private final MutableSet listenedPartitions = Sets.mutable.of().asSynchronized(); - - private volatile boolean active; - - private final PartitionToFetchRequestFunction partitionToFetchRequestFunction = - new PartitionToFetchRequestFunction(); - - private final IsLeaderErrorPredicate isLeaderPredicate = new IsLeaderErrorPredicate(); - - private final IsOffsetOutOfRangePredicate offsetOutOfRangePredicate = new IsOffsetOutOfRangePredicate(); - - public FetchTask(BrokerAddress brokerAddress, RichIterable initialPartitions) { - this.brokerAddress = brokerAddress; - this.active = true; - this.listenedPartitions.addAll(initialPartitions.toSet()); - } - - @Override - public boolean isLongLived() { - return true; - } - - public boolean addListenedPartitionsIfActive(Iterable partitions) { - synchronized (listenedPartitions) { - if (active) { - listenedPartitions.addAllIterable(partitions); - } - return active; - } - } - - @Override - public void run() { - try { - while (active && isRunning()) { - synchronized (listenedPartitions) { - try { - if (!listenedPartitions.isEmpty()) { - Result result = fetchAvailableData(); - handleSuccessful(result); - if (result.getErrors().size() > 0) { - handleErrors(result); - } - } - else { - active = false; - } - } - catch (ConsumerException e) { - active = false; - // the connection is broken, terminate the task - kafkaTemplate.getConnectionFactory().disconnect(brokerAddress); - resetLeaders(listenedPartitions.toImmutable()); - } - } - } - } - finally { - active = false; - synchronized (fetchTasksByBroker) { - if (fetchTasksByBroker.get(brokerAddress) == this) { - fetchTasksByBroker.remove(brokerAddress); - } - } - } - } - - private Result fetchAvailableData() { - return kafkaTemplate.receive(listenedPartitions.collect(partitionToFetchRequestFunction)); - } - - private void handleSuccessful(Result result) { - Iterable batches = result.getResults().values(); - for (KafkaMessageBatch batch : batches) { - if (!batch.getMessages().isEmpty()) { - long highestFetchedOffset = 0; - for (KafkaMessage kafkaMessage : batch.getMessages()) { - // fetch operations may return entire blocks of compressed messages, - // which may have lower offsets than the ones requested - // thus a batch may contain messages that have been processed already - if (kafkaMessage.getMetadata().getOffset() >= fetchOffsets.get(batch.getPartition())) { - messageDispatcher.dispatch(kafkaMessage); - } - highestFetchedOffset = Math.max(highestFetchedOffset, kafkaMessage.getMetadata().getNextOffset()); - } - fetchOffsets.replace(batch.getPartition(), highestFetchedOffset); - } - } - } - - private void handleErrors(Result result) { - Map errors = result.getErrors(); - PartitionIterable> splitByLeaderError = - Iterate.partition(errors.entrySet(), isLeaderPredicate); - RichIterable partitionsWithLeaderErrors = splitByLeaderError.getSelected() - .collect(Functions.getKeyFunction()); - resetLeaders(partitionsWithLeaderErrors); - PartitionIterable> splitByOffsetError = - splitByLeaderError.getRejected().partition(offsetOutOfRangePredicate); - RichIterable partitionsWithWrongOffsets = - splitByOffsetError.getSelected().collect(Functions.getKeyFunction()); - resetOffsets(partitionsWithWrongOffsets.toSet()); - // it's not a leader issue, remove everything else - RichIterable remainingPartitionsWithErrors - = splitByOffsetError.getRejected().collect(Functions.getKeyFunction()); - listenedPartitions.removeAllIterable(remainingPartitionsWithErrors); - } - - private void resetLeaders(final Iterable partitionsToReset) { - listenedPartitions.removeAllIterable(partitionsToReset); - adminTaskExecutor.execute(new UpdateLeadersTask(partitionsToReset)); - } - - private void resetOffsets(final Collection partitionsToResetOffsets) { - listenedPartitions.removeAllIterable(partitionsToResetOffsets); - adminTaskExecutor.execute(new UpdateOffsetsTask(partitionsToResetOffsets)); - } - - private class UpdateLeadersTask implements SchedulingAwareRunnable { - - private final Iterable partitionsToReset; - - public UpdateLeadersTask(Iterable partitionsToReset) { - this.partitionsToReset = partitionsToReset; - } - - @Override - public boolean isLongLived() { - return true; - } - - @Override - public void run() { - // fetch can complete successfully or unsuccessfully - boolean fetchCompleted = false; - while (!fetchCompleted && isRunning()) { - try { - FastList partitionsAsList = FastList.newList(partitionsToReset); - FastList topics = partitionsAsList.collect(new PartitionToTopicFunction()).distinct(); - kafkaTemplate.getConnectionFactory().refreshMetadata(topics); - - MutableSetMultimap partitionsByBroker = UnifiedMap - .newMap(kafkaTemplate.getConnectionFactory().getLeaders(partitionsToReset)).flip(); - for (Pair> pair : partitionsByBroker - .keyMultiValuePairsView()) { - synchronized (fetchTasksByBroker) { - boolean addedSuccessfully = false; - FetchTask fetchTask = fetchTasksByBroker.get(pair.getOne()); - if (fetchTask != null) { - addedSuccessfully = fetchTask.addListenedPartitionsIfActive(pair.getTwo()); - } - if (!addedSuccessfully) { - fetchTask = new FetchTask(pair.getOne(), pair.getTwo()); - fetchTaskExecutor.execute(fetchTask); - fetchTasksByBroker.put(pair.getOne(), fetchTask); - } - } - } - fetchCompleted = true; - } - catch (Exception e) { - if (isRunning()) { - try { - Thread.sleep(DEFAULT_WAIT_FOR_LEADER_REFRESH_RETRY); - } - catch (InterruptedException e1) { - Thread.currentThread().interrupt(); - log.error("Interrupted after refresh leaders failure for: " + Iterate - .makeString(partitionsToReset, ",")); - fetchCompleted = true; - } - } - } - } - } - - } - - private class UpdateOffsetsTask implements Runnable { - - private final Collection partitionsToResetOffsets; - - public UpdateOffsetsTask(Collection partitionsToResetOffsets) { - this.partitionsToResetOffsets = partitionsToResetOffsets; - } - - @Override - public void run() { - offsetManager.resetOffsets(partitionsToResetOffsets); - for (Partition partition : partitionsToResetOffsets) { - fetchOffsets.replace(partition, offsetManager.getOffset(partition)); - } - synchronized (fetchTasksByBroker) { - boolean addedSuccessfully = false; - FetchTask fetchTask = fetchTasksByBroker.get(brokerAddress); - if (fetchTask != null) { - addedSuccessfully = fetchTask.addListenedPartitionsIfActive(partitionsToResetOffsets); - } - if (!addedSuccessfully) { - fetchTask = new FetchTask(brokerAddress, Sets.immutable.ofAll(partitionsToResetOffsets)); - fetchTaskExecutor.execute(fetchTask); - fetchTasksByBroker.put(brokerAddress, fetchTask); - } - } - } - - } - - @SuppressWarnings("serial") - private class IsLeaderErrorPredicate implements Predicate> { - - @Override - public boolean accept(Map.Entry each) { - return each.getValue() == ErrorMapping.NotLeaderForPartitionCode() || each.getValue() == ErrorMapping - .UnknownTopicOrPartitionCode(); - } - - } - - @SuppressWarnings("serial") - private class IsOffsetOutOfRangePredicate implements Predicate> { - - @Override - public boolean accept(Map.Entry each) { - return each.getValue() == ErrorMapping.OffsetOutOfRangeCode(); - } - - } - } - - @SuppressWarnings("serial") - class GetOffsetForPartitionFunction extends CheckedFunction { - - @Override - public Long safeValueOf(Partition object) throws Exception { - try { - return offsetManager.getOffset(object); - } - catch (Exception e) { - log.error(e); - throw e; - } - } - - } - - @SuppressWarnings("serial") - private class PartitionToLeaderFunction implements Function { - - @Override - public BrokerAddress valueOf(Partition partition) { - return kafkaTemplate.getConnectionFactory().getLeader(partition); - } - - } - - @SuppressWarnings("serial") - private class PartitionToFetchRequestFunction implements Function { - - @Override - public FetchRequest valueOf(Partition partition) { - return new FetchRequest(partition, fetchOffsets.get(partition), maxFetch); - } - - } - - @SuppressWarnings("serial") - static class GetPartitionsForTopic extends CheckedFunction> { - - private final ConnectionFactory connectionFactory; - - public GetPartitionsForTopic(ConnectionFactory connectionFactory) { - this.connectionFactory = connectionFactory; - } - - @Override - public Iterable safeValueOf(String topic) throws Exception { - return connectionFactory.getPartitions(topic); - } - - } - - @SuppressWarnings("serial") - private class PartitionToTopicFunction implements Function { - - @Override - public String valueOf(Partition object) { - return object.getTopic(); - } - - } - -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/KafkaNativeOffsetManager.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/KafkaNativeOffsetManager.java deleted file mode 100644 index 8725d6d173..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/KafkaNativeOffsetManager.java +++ /dev/null @@ -1,231 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.listener; - -import java.io.IOException; -import java.util.Collections; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -import org.I0Itec.zkclient.ZkClient; - -import org.springframework.beans.factory.InitializingBean; -import org.springframework.integration.kafka.core.BrokerAddress; -import org.springframework.integration.kafka.core.Configuration; -import org.springframework.integration.kafka.core.Connection; -import org.springframework.integration.kafka.core.ConnectionFactory; -import org.springframework.integration.kafka.core.ConsumerException; -import org.springframework.integration.kafka.core.DefaultConnectionFactory; -import org.springframework.integration.kafka.core.KafkaConsumerDefaults; -import org.springframework.integration.kafka.core.Partition; -import org.springframework.integration.kafka.core.PartitionNotFoundException; -import org.springframework.integration.kafka.core.Result; -import org.springframework.integration.kafka.support.ZookeeperConnect; -import org.springframework.retry.RetryCallback; -import org.springframework.retry.RetryContext; -import org.springframework.retry.backoff.ExponentialBackOffPolicy; -import org.springframework.retry.listener.RetryListenerSupport; -import org.springframework.retry.policy.SimpleRetryPolicy; -import org.springframework.retry.support.RetryTemplate; -import org.springframework.util.Assert; - -import com.gs.collections.impl.factory.Maps; - -import kafka.client.ClientUtils$; -import kafka.common.ErrorMapping; -import kafka.common.OffsetAndMetadata; -import kafka.network.BlockingChannel; -import kafka.utils.ZKStringSerializer$; - -/** - * Implementation of an {@link OffsetManager} that uses kafka native 'topic' offset storage. - * - * @author Chris Lemper - * @author Marius Bogoevici - * @since 1.3 - */ -public class KafkaNativeOffsetManager extends AbstractOffsetManager implements InitializingBean { - - private static final String PARTITION_ATTRIBUTE = "partition"; - - private final Map offsetManagerBrokerAddressCache = new ConcurrentHashMap<>(); - - private final ZkClient zkClient; - - private RetryTemplate retryTemplate; - - /** - * @param connectionFactory a Kafka connection factory - * @param zookeeperConnect the zookeeper connection information - */ - public KafkaNativeOffsetManager(ConnectionFactory connectionFactory, ZookeeperConnect zookeeperConnect) { - this(connectionFactory, zookeeperConnect, Collections.emptyMap()); - } - - /** - * @param connectionFactory a Kafka connection factory - * @param zookeeperConnect zookeeper connection for retrieving - * @param initialOffsets a map of partitions to initial offsets - */ - public KafkaNativeOffsetManager(ConnectionFactory connectionFactory, ZookeeperConnect zookeeperConnect, - Map initialOffsets) { - super(connectionFactory, initialOffsets); - Assert.notNull(zookeeperConnect, "'zookeeperConnect' must not be null."); - this.zkClient = new ZkClient(zookeeperConnect.getZkConnect(), - Integer.parseInt(zookeeperConnect.getZkSessionTimeout()), - Integer.parseInt(zookeeperConnect.getZkConnectionTimeout()), - ZKStringSerializer$.MODULE$); - } - - public void setRetryTemplate(RetryTemplate retryTemplate) { - this.retryTemplate = retryTemplate; - } - - @Override - public void afterPropertiesSet() throws Exception { - if (this.retryTemplate == null) { - this.retryTemplate = new RetryTemplate(); - this.retryTemplate.registerListener(new ResetOffsetManagerBrokerAddressRetryListener()); - final SimpleRetryPolicy retryPolicy = new SimpleRetryPolicy(); - retryPolicy.setMaxAttempts(5); - this.retryTemplate.setRetryPolicy(retryPolicy); - final ExponentialBackOffPolicy backOffPolicy = new ExponentialBackOffPolicy(); - backOffPolicy.setInitialInterval(125L); - backOffPolicy.setMaxInterval(5000L); - backOffPolicy.setMultiplier(2); - this.retryTemplate.setBackOffPolicy(backOffPolicy); - } - } - - @Override - protected Long doGetOffset(final Partition partition) { - final Long offset = this.retryTemplate.execute(new RetryCallback() { - - @Override - public Long doWithRetry(RetryContext context) throws RuntimeException { - context.setAttribute(PARTITION_ATTRIBUTE, partition); - Result result = getOffsetManagerConnection(partition).fetchStoredOffsetsForConsumer( - getConsumerId(), partition); - checkResultForErrors(result, partition); - return result.getResult(partition); - } - - @Override - public String toString() { - return String.format("fetchStoredOffsetsForConsumer(%s, %s)", getConsumerId(), partition); - } - - }); - - if (offset != null && offset < 0) { - return null; - } - - return offset; - } - - @Override - protected void doUpdateOffset(final Partition partition, final long offset) { - this.retryTemplate.execute(new RetryCallback() { - - @Override - public Void doWithRetry(RetryContext context) throws RuntimeException { - context.setAttribute(PARTITION_ATTRIBUTE, partition); - Result result = getOffsetManagerConnection(partition).commitOffsetsForConsumer( - getConsumerId(), Maps.immutable.of(partition, offset).castToMap()); - checkResultForErrors(result, partition); - return null; - } - - @Override - public String toString() { - return String.format("commitOffsetsForConsumer(%s, %s, %s)", getConsumerId(), partition, offset); - } - - }); - } - - @Override - protected void doRemoveOffset(Partition partition) { - doUpdateOffset(partition, OffsetAndMetadata.InvalidOffset()); - } - - @Override - public void close() throws IOException { - this.zkClient.close(); - } - - @Override - public void flush() throws IOException { - // this function left intentionally blank - } - - private BrokerAddress getOffsetManagerBrokerAddress(final Partition partition) { - BrokerAddress brokerAddress = this.offsetManagerBrokerAddressCache.get(partition); - if (brokerAddress == null) { - int socketTimeoutMs = KafkaConsumerDefaults.SOCKET_TIMEOUT_INT; - int retryBackOffMs = KafkaConsumerDefaults.BACKOFF_INCREMENT_INT; - if (this.connectionFactory instanceof DefaultConnectionFactory) { - Configuration configuration = ((DefaultConnectionFactory) connectionFactory).getConfiguration(); - socketTimeoutMs = configuration.getSocketTimeout(); - retryBackOffMs = configuration.getBackOff(); - } - final BlockingChannel channel = ClientUtils$.MODULE$.channelToOffsetManager( - getConsumerId(), zkClient, socketTimeoutMs, retryBackOffMs); - brokerAddress = new BrokerAddress(channel.host(), channel.port()); - if (log.isDebugEnabled()) { - log.debug(String.format("Offset manager for [%s] is at [%s].", partition, - brokerAddress)); - } - this.offsetManagerBrokerAddressCache.put(partition, brokerAddress); - channel.disconnect(); - } - return brokerAddress; - } - - private Connection getOffsetManagerConnection(final Partition partition) { - return this.connectionFactory.connect(getOffsetManagerBrokerAddress(partition)); - } - - private void checkResultForErrors(final Result result, final Partition partition) { - if (result.getErrors().containsKey(partition)) { - final short errorCode = result.getError(partition); - if (errorCode == ErrorMapping.UnknownTopicOrPartitionCode()) { - throw new PartitionNotFoundException(partition); - } - else if (errorCode != ErrorMapping.NoError()) { - throw new ConsumerException(ErrorMapping.exceptionFor(errorCode)); - } - } - } - - private class ResetOffsetManagerBrokerAddressRetryListener extends RetryListenerSupport { - - @Override - @SuppressWarnings("unchecked") - public void onError(RetryContext context, RetryCallback callback, Throwable t) { - if (log.isWarnEnabled()) { - log.warn("Retrying kafka operation [" + callback + "] due to [" + t + "]", t); - } - Partition partition = (Partition) context.getAttribute(PARTITION_ATTRIBUTE); - if (partition != null) { - offsetManagerBrokerAddressCache.remove(partition); - } - } - } - -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/KafkaTopicOffsetManager.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/KafkaTopicOffsetManager.java deleted file mode 100644 index 7a8856d9e5..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/KafkaTopicOffsetManager.java +++ /dev/null @@ -1,506 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.listener; - -import java.io.IOException; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.ConcurrentMap; - -import kafka.admin.AdminUtils$; -import kafka.api.OffsetRequest; -import kafka.common.ErrorMapping$; -import kafka.common.TopicExistsException; -import kafka.serializer.Decoder; -import kafka.utils.ZKStringSerializer$; -import org.I0Itec.zkclient.ZkClient; -import org.I0Itec.zkclient.exception.ZkInterruptedException; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.kafka.clients.producer.Producer; -import org.apache.kafka.clients.producer.ProducerConfig; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.common.serialization.Serializer; - -import org.springframework.beans.factory.BeanInitializationException; -import org.springframework.beans.factory.InitializingBean; -import org.springframework.integration.kafka.core.BrokerAddress; -import org.springframework.integration.kafka.core.DefaultConnectionFactory; -import org.springframework.integration.kafka.core.FetchRequest; -import org.springframework.integration.kafka.core.KafkaMessage; -import org.springframework.integration.kafka.core.KafkaMessageBatch; -import org.springframework.integration.kafka.core.KafkaTemplate; -import org.springframework.integration.kafka.core.Partition; -import org.springframework.integration.kafka.core.Result; -import org.springframework.integration.kafka.core.TopicNotFoundException; -import org.springframework.integration.kafka.core.ZookeeperConfiguration; -import org.springframework.integration.kafka.support.ProducerFactoryBean; -import org.springframework.integration.kafka.support.ProducerMetadata; -import org.springframework.integration.kafka.support.ZookeeperConnect; -import org.springframework.integration.kafka.util.LoggingUtils; -import org.springframework.integration.kafka.util.MessageUtils; -import org.springframework.retry.RetryCallback; -import org.springframework.retry.RetryContext; -import org.springframework.retry.backoff.ExponentialBackOffPolicy; -import org.springframework.retry.policy.SimpleRetryPolicy; -import org.springframework.retry.support.RetryTemplate; -import org.springframework.util.Assert; -import org.springframework.util.ObjectUtils; - -/** - * Implementation of an {@link OffsetManager} that uses a Kafka topic as the underlying support. - * For its proper functioning, the Kafka server(s) must set {@code log.cleaner.enable=true}. It relies on the property - * {@code cleanup.policy=compact} to be set on the target topic, and if the topic is not found, - * it will create a topic with the appropriate settings. - * - * @author Marius Bogoevici - * @author Artem Bilan - */ -public class KafkaTopicOffsetManager extends AbstractOffsetManager implements InitializingBean { - - private static final KeySerializerDecoder KEY_CODEC = new KeySerializerDecoder(); - - private static final LongSerializerDecoder VALUE_CODEC = new LongSerializerDecoder(); - - public static final String CLEANUP_POLICY = "cleanup.policy"; - - public static final String CLEANUP_POLICY_COMPACT = "compact"; - - public static final String DELETE_RETENTION = "delete.retention.ms"; - - public static final String SEGMENT_BYTES = "segment.bytes"; - - private final ZookeeperConnect zookeeperConnect; - - private final String topic; - - private final KafkaTemplate kafkaTemplate; - - private final ConcurrentMap data = new ConcurrentHashMap(); - - private ProducerMetadata.CompressionType compressionType = ProducerMetadata.CompressionType.none; - - private Producer producer; - - private int maxSize = 10 * 1024; - - private int segmentSize = 25 * 1024; - - private int retentionTime = 60000; - - private int replicationFactor; - - private int batchBytes = 200; - - private int requiredAcks = 1; - - private int maxQueueBufferingTime; - - public KafkaTopicOffsetManager(ZookeeperConnect zookeeperConnect, String topic) { - this(zookeeperConnect, topic, new HashMap()); - } - - public KafkaTopicOffsetManager(ZookeeperConnect zookeeperConnect, String topic, - Map initialOffsets) { - super(new DefaultConnectionFactory(new ZookeeperConfiguration(zookeeperConnect)), initialOffsets); - Assert.notNull(zookeeperConnect); - this.zookeeperConnect = zookeeperConnect; - this.kafkaTemplate = new KafkaTemplate(connectionFactory); - this.topic = topic; - } - - /** - * Sets the maximum size of a fetch request, allowing to tune the initialization process. - * - * @param maxSize the maximum amount of data to be brought on a fetch - */ - public void setMaxSize(int maxSize) { - this.maxSize = maxSize; - } - - /** - * The compression type for writing to the offset topic - * - * @param compressionType the compression type - */ - public void setCompressionCodec(ProducerMetadata.CompressionType compressionType) { - this.compressionType = compressionType; - } - - /** - * For how long will producers buffer data before writing to the topic - * - * @param maxQueueBufferingTime the maximum buffering window (in milliseconds) - */ - public void setMaxQueueBufferingTime(int maxQueueBufferingTime) { - this.maxQueueBufferingTime = maxQueueBufferingTime; - } - - /** - * The size of a segment in the offset topic - * - * @param segmentSize the segment size of an offset topic - */ - public void setSegmentSize(int segmentSize) { - this.segmentSize = segmentSize; - } - - /** - * How long are dead records retained in the offset topic - * - * @param retentionTime the retention time for dead records (in seconds) - */ - public void setRetentionTime(int retentionTime) { - this.retentionTime = retentionTime; - } - - /** - * The replication factor of the offset topic - * - * @param replicationFactor the replication factor - */ - public void setReplicationFactor(int replicationFactor) { - this.replicationFactor = replicationFactor; - } - - /** - * The maximum batch size in bytes for offset writes - * - * @param batchBytes maximum batching window - */ - public void setBatchBytes(int batchBytes) { - this.batchBytes = batchBytes; - } - - /** - * The number of required acks on write operations - * - * @param requiredAcks the number of required acks - */ - public void setRequiredAcks(int requiredAcks) { - this.requiredAcks = requiredAcks; - } - - @Override - public void afterPropertiesSet() throws Exception { - ((DefaultConnectionFactory) this.connectionFactory).afterPropertiesSet(); - ZkClient zkClient = new ZkClient(this.zookeeperConnect.getZkConnect(), - Integer.parseInt(this.zookeeperConnect.getZkSessionTimeout()), - Integer.parseInt(this.zookeeperConnect.getZkConnectionTimeout()), - ZKStringSerializer$.MODULE$); - try { - createCompactedTopicIfNotFound(zkClient); - validateOffsetTopic(zkClient); - Partition offsetPartition = new Partition(this.topic, 0); - BrokerAddress offsetPartitionLeader = this.connectionFactory.getLeader(offsetPartition); - readOffsetData(offsetPartition, offsetPartitionLeader); - initializeProducer(offsetPartitionLeader); - } - finally { - try { - zkClient.close(); - } - catch (ZkInterruptedException e) { - log.error("Error while closing Zookeeper client", e); - } - } - } - - @Override - protected void doUpdateOffset(Partition partition, long offset) { - this.data.put(partition, offset); - this.producer.send(new ProducerRecord<>(this.topic, new Key(this.consumerId, partition), offset)); - } - - @Override - protected void doRemoveOffset(Partition partition) { - this.data.remove(partition); - this.producer.send(new ProducerRecord(this.topic, new Key(this.consumerId, partition), null)); - } - - @Override - protected Long doGetOffset(Partition partition) { - return this.data.get(partition); - } - - @Override - public void flush() throws IOException { - // not supported - } - - @Override - public void close() throws IOException { - this.producer.close(); - try { - ((DefaultConnectionFactory) this.connectionFactory).destroy(); - } - catch (Exception e) { - throw new IOException(e); - } - } - - - private void createCompactedTopicIfNotFound(ZkClient zkClient) { - Properties topicConfig = new Properties(); - topicConfig.setProperty(CLEANUP_POLICY, CLEANUP_POLICY_COMPACT); - topicConfig.setProperty(DELETE_RETENTION, String.valueOf(this.retentionTime)); - topicConfig.setProperty(SEGMENT_BYTES, String.valueOf(this.segmentSize)); - try { - this.replicationFactor = 1; - AdminUtils$.MODULE$.createTopic(zkClient, this.topic, 1, this.replicationFactor, topicConfig); - } - catch (TopicExistsException e) { - log.debug("Topic already exists", e); - } - } - - private void validateOffsetTopic(ZkClient zkClient) throws Exception { - //validate that the topic exists, but also prevent working with the topic until it's fully initialized - // set a retry template, since operations may fail - RetryTemplate retryValidateTopic = new RetryTemplate(); - retryValidateTopic.setRetryPolicy(new SimpleRetryPolicy(10, - Collections., Boolean>singletonMap(TopicNotFoundException.class, true))); - ExponentialBackOffPolicy backOffPolicy = new ExponentialBackOffPolicy(); - backOffPolicy.setInitialInterval(100L); - backOffPolicy.setMaxInterval(1000L); - backOffPolicy.setMultiplier(2); - retryValidateTopic.setBackOffPolicy(backOffPolicy); - - Collection partitions = - retryValidateTopic.execute(new RetryCallback, Exception>() { - - @Override - public Collection doWithRetry(RetryContext context) throws Exception { - return connectionFactory.getPartitions(topic); - } - - }); - - if (partitions.size() > 1) { - throw new BeanInitializationException("Offset management topic cannot have more than one partition"); - } - - Properties properties = AdminUtils$.MODULE$.fetchTopicConfig(zkClient, this.topic); - if (!properties.containsKey(CLEANUP_POLICY) - || !CLEANUP_POLICY_COMPACT.equals(properties.getProperty(CLEANUP_POLICY))) { - // we set the property to compact, but if using an already created topic, - // we must check if it is set up correctly - throw new BeanInitializationException("Property 'cleanup.policy' must be set to 'compact' on offset topic"); - } - } - - private void readOffsetData(Partition offsetManagementPartition, BrokerAddress leader) { - Result earliestOffsetResult = this.connectionFactory.connect(leader) - .fetchInitialOffset(OffsetRequest.EarliestTime(), offsetManagementPartition); - if (earliestOffsetResult.getErrors().size() > 0) { - throw new BeanInitializationException("Cannot initialize offset manager, unable to read earliest offset", - ErrorMapping$.MODULE$.exceptionFor(earliestOffsetResult.getError(offsetManagementPartition))); - } - Result latestOffsetResult = this.connectionFactory.connect(leader) - .fetchInitialOffset(OffsetRequest.LatestTime(), offsetManagementPartition); - if (latestOffsetResult.getErrors().size() > 0) { - throw new BeanInitializationException("Cannot initialize offset manager, unable to read latest offset"); - } - - long initialOffset = earliestOffsetResult.getResult(offsetManagementPartition); - long finalOffset = latestOffsetResult.getResult(offsetManagementPartition); - - // read repeatedly until we drain the topic and add messages to the data map - long readingOffset = initialOffset; - while (readingOffset < finalOffset) { - FetchRequest fetchRequest = new FetchRequest(offsetManagementPartition, readingOffset, maxSize); - Result receive = this.kafkaTemplate.receive(Collections.singleton(fetchRequest)); - if (receive.getErrors().size() > 0) { - throw new BeanInitializationException("Error while fetching initial offsets:", - ErrorMapping$.MODULE$.exceptionFor(receive.getError(offsetManagementPartition))); - } - KafkaMessageBatch result = receive.getResult(offsetManagementPartition); - for (KafkaMessage kafkaMessage : result.getMessages()) { - checkAndAddData(kafkaMessage); - readingOffset = kafkaMessage.getMetadata().getNextOffset(); - } - if (log.isDebugEnabled()) { - log.debug(data.size() + " entries in the final map"); - } - if (log.isTraceEnabled()) { - for (Map.Entry dataEntry : data.entrySet()) { - log.trace(String.format("Final value for %s : %s", dataEntry.getKey().toString(), - String.valueOf(dataEntry.getValue()))); - } - } - } - } - - private void checkAndAddData(KafkaMessage kafkaMessage) { - Key key = MessageUtils.decodeKey(kafkaMessage, KEY_CODEC); - Long value = MessageUtils.decodePayload(kafkaMessage, VALUE_CODEC); - if (log.isTraceEnabled()) { - log.trace("Loading key " + key + " with value " + value); - } - // we are only interested for messages that are intended for this consumer id - if (key != null && ObjectUtils.nullSafeEquals(this.consumerId, key.getConsumerId())) { - if (null != value) { - // write data in the cache, overwriting the older values - this.data.put(key.getPartition(), value); - } - else { - // a null value means that the data has been deleted, but not compacted yet - if (this.data.containsKey(key.getPartition())) { - this.data.remove(key.getPartition()); - } - } - } - } - - - private void initializeProducer(BrokerAddress leader) throws Exception { - ProducerMetadata producerMetadata = new ProducerMetadata<>(this.topic, Key.class, Long.class, - KEY_CODEC, VALUE_CODEC); - producerMetadata.setBatchBytes(batchBytes); - producerMetadata.setCompressionType(compressionType); - Properties additionalProps = new Properties(); - additionalProps.setProperty(ProducerConfig.LINGER_MS_CONFIG, Integer.toString(maxQueueBufferingTime)); - additionalProps.setProperty(ProducerConfig.ACKS_CONFIG, Integer.toString(requiredAcks)); - ProducerFactoryBean producerFactoryBean = new ProducerFactoryBean<>(producerMetadata, - leader.toString(), additionalProps); - this.producer = producerFactoryBean.getObject(); - } - - private static int intFromBytes(byte[] bytes, int start) { - return bytes[start] << 24 | (bytes[start + 1] & 0xFF) << 16 - | (bytes[start + 2] & 0xFF) << 8 | (bytes[start + 3] & 0xFF); - } - - private static byte[] intToBytes(Integer message) { - int value = message; - return new byte[] { - (byte) (value >>> 24), - (byte) (value >>> 16), - (byte) (value >>> 8), - (byte) value - }; - } - - /** - * Wraps the partition and consumer information and will be used as a key on the Kafka topic - */ - public static class Key { - - String consumerId; - - Partition partition; - - public Key(String consumerID, Partition partition) { - Assert.notNull(consumerID, "Consumer Id cannot be null"); - Assert.notNull(partition, "Partition cannot be null"); - this.consumerId = consumerID; - this.partition = partition; - } - - public String getConsumerId() { - return this.consumerId; - } - - public Partition getPartition() { - return this.partition; - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - - Key key = (Key) o; - - return this.consumerId.equals(key.consumerId) && this.partition.equals(key.partition); - - } - - @Override - public int hashCode() { - int result = this.consumerId.hashCode(); - result = 31 * result + this.partition.hashCode(); - return result; - } - - } - - public static class KeySerializerDecoder implements Serializer, Decoder { - - private static final Log log = LogFactory.getLog(KeySerializerDecoder.class); - - @Override - public Key fromBytes(byte[] bytes) { - if (bytes == null || bytes.length <= 0) { - return null; - } - try { - // calculate the offsets in the key array - int consumerIdSize = intFromBytes(bytes, 0); - int topicIdSize = intFromBytes(bytes, consumerIdSize + 4); - // reconstruct the key - return new Key(new String(bytes, 4, consumerIdSize), - new Partition(new String(bytes, consumerIdSize + 8, topicIdSize), - intFromBytes(bytes, consumerIdSize + topicIdSize + 8))); - } - catch (Exception e) { - if (log.isDebugEnabled()) { - log.debug("Cannot decode key:" + LoggingUtils.asCommaSeparatedHexDump(bytes)); - } - return null; - } - } - - @Override - public void configure(Map configs, boolean isKey) { - // no-op - } - - @Override - public byte[] serialize(String topic, Key data) { - if (data == null) { - return null; - } - try { - byte[] consumerIdBytes = data.consumerId.getBytes("UTF-8"); - byte[] topicNameBytes = data.partition.getTopic().getBytes("UTF-8"); - byte[] partitionIdBytes = intToBytes(data.partition.getId()); - byte[] result = new byte[4 + consumerIdBytes.length + 4 + topicNameBytes.length + 4]; - System.arraycopy(intToBytes(consumerIdBytes.length), 0, result, 0, 4); - System.arraycopy(consumerIdBytes, 0, result, 4, consumerIdBytes.length); - System.arraycopy(intToBytes(topicNameBytes.length), 0, result, consumerIdBytes.length + 4, 4); - System.arraycopy(topicNameBytes, 0, result, consumerIdBytes.length + 8, topicNameBytes.length); - System.arraycopy(partitionIdBytes, 0, result, consumerIdBytes.length + topicNameBytes.length + 8, 4); - return result; - } - catch (Exception e) { - throw new RuntimeException(e); - } - } - - @Override - public void close() { - //no-op - } - - } - -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/LoggingErrorHandler.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/LoggingErrorHandler.java deleted file mode 100644 index c149a354fb..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/LoggingErrorHandler.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package org.springframework.integration.kafka.listener; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import org.springframework.integration.kafka.core.KafkaMessage; -import org.springframework.util.ObjectUtils; - -/** - * @author Marius Bogoevici - */ -public class LoggingErrorHandler implements ErrorHandler { - - private static final Log log = LogFactory.getLog(LoggingErrorHandler.class); - - @Override - public void handle(Exception thrownException, KafkaMessage message) { - log.error("Error while processing: " + ObjectUtils.nullSafeToString(message), thrownException); - } - -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/LongSerializerDecoder.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/LongSerializerDecoder.java deleted file mode 100644 index 258b63f5ad..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/LongSerializerDecoder.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.listener; - -import java.nio.ByteBuffer; -import java.util.Map; - -import kafka.serializer.Decoder; -import kafka.serializer.Encoder; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.kafka.common.serialization.Serializer; - -import org.springframework.integration.kafka.util.LoggingUtils; - -/** - * Kafka {@link Encoder} and {@link Decoder} for Long values. - * - * @author Marius Bogoevici - */ -public class LongSerializerDecoder implements Serializer, Decoder { - - private Log log = LogFactory.getLog(LongSerializerDecoder.class); - - @Override - public Long fromBytes(byte[] bytes) { - if (bytes == null || bytes.length <= 0) { - return null; - } - else { - try { - return ByteBuffer.wrap(bytes).getLong(0); - } - catch (Exception e) { - if (log.isDebugEnabled()) { - log.debug("Cannot decode value: " + LoggingUtils.asCommaSeparatedHexDump(bytes)); - } - return null; - } - } - } - - @Override - public void configure(Map configs, boolean isKey) { - // no-op - } - - @Override - public byte[] serialize(String topic, Long data) { - if (data == null) { - return null; - } - else { - return ByteBuffer.allocate(8).putLong(data).array(); - } - } - - @Override - public void close() { - // no-op - } -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/MessageListener.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/MessageListener.java deleted file mode 100644 index 554df9efef..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/MessageListener.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package org.springframework.integration.kafka.listener; - -import org.springframework.integration.kafka.core.KafkaMessage; - -/** - * Listener for handling incoming Kafka messages - * - * @author Marius Bogoevici - */ -public interface MessageListener { - - /** - * Executes when a Kafka message is received - * @param message the Kafka message to be processed - */ - void onMessage(KafkaMessage message); - -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/MetadataStoreOffsetManager.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/MetadataStoreOffsetManager.java deleted file mode 100644 index 3b3ed615f8..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/MetadataStoreOffsetManager.java +++ /dev/null @@ -1,99 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.listener; - -import java.io.Closeable; -import java.io.Flushable; -import java.io.IOException; -import java.util.Map; - -import org.springframework.integration.kafka.core.ConnectionFactory; -import org.springframework.integration.kafka.core.Partition; -import org.springframework.integration.metadata.MetadataStore; -import org.springframework.integration.metadata.SimpleMetadataStore; - -/** - * An {@link OffsetManager} that persists offsets into a {@link MetadataStore}. - * - * @author Marius Bogoevici - */ -public class MetadataStoreOffsetManager extends AbstractOffsetManager { - - private MetadataStore metadataStore = new SimpleMetadataStore(); - - public MetadataStoreOffsetManager(ConnectionFactory connectionFactory) { - super(connectionFactory); - } - - public MetadataStoreOffsetManager(ConnectionFactory connectionFactory, Map initialOffsets) { - super(connectionFactory, initialOffsets); - } - - /** - * The backing {@link MetadataStore} for storing offsets. - * - * @param metadataStore a fully configured {@link MetadataStore} instance - */ - public void setMetadataStore(MetadataStore metadataStore) { - this.metadataStore = metadataStore; - } - - @Override - public void close() throws IOException { - // Flush before closing. This may be redundant, but ensures that the metadata store is closed properly - flush(); - if (this.metadataStore instanceof Closeable) { - ((Closeable) this.metadataStore).close(); - } - } - - @Override - public void flush() throws IOException { - if (this.metadataStore instanceof Flushable) { - ((Flushable) this.metadataStore).flush(); - } - } - - @Override - protected void doUpdateOffset(Partition partition, long offset) { - this.metadataStore.put(generateKey(partition), Long.toString(offset)); - } - - @Override - protected void doRemoveOffset(Partition partition) { - this.metadataStore.remove(generateKey(partition)); - } - - protected Long doGetOffset(Partition partition) { - String storedOffsetValueAsString = this.metadataStore.get(generateKey(partition)); - Long storedOffsetValue = null; - if (storedOffsetValueAsString != null) { - try { - storedOffsetValue = Long.parseLong(storedOffsetValueAsString); - } - catch (NumberFormatException e) { - log.warn("Invalid value: " + storedOffsetValueAsString); - } - } - return storedOffsetValue; - } - - public String generateKey(Partition partition) { - return partition.getTopic() + ":" + partition.getId() + ":" + getConsumerId(); - } - -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/OffsetManager.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/OffsetManager.java deleted file mode 100644 index 3d93c17594..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/OffsetManager.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.listener; - -import java.io.Closeable; -import java.io.Flushable; -import java.util.Collection; - -import org.springframework.integration.kafka.core.Partition; - -/** - * Stores and retrieves offsets for a Kafka consumer - * - * @author Marius Bogoevici - */ -public interface OffsetManager extends Closeable, Flushable { - - /** - * Updates the offset for a given {@link Partition} - * @param partition the partition whose offset is to be updated - * @param offset the new offset value - */ - void updateOffset(Partition partition, long offset); - - /** - * Retrieves the offset for a given {@link Partition} - * @param partition the partition to be - * @return the offset value - */ - long getOffset(Partition partition); - - /** - * Removes the offset for a given {@link Partition}. Useful - * for components that need to clean up after themselves. - * @param partition for which to delete the JavaDoc - * - */ - void deleteOffset(Partition partition); - - /** - * Resets offsets for the given {@link Partition}s. To be invoked when the values stored are invalid, - * so a client cannot resume from that position. Implementations must decide on the best strategy to follow. - * @param partition to reset - */ - void resetOffsets(Collection partition); - -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/QueueingMessageListenerInvoker.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/QueueingMessageListenerInvoker.java deleted file mode 100644 index 7c61bcea57..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/QueueingMessageListenerInvoker.java +++ /dev/null @@ -1,264 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.listener; - -import java.util.List; -import java.util.concurrent.AbstractExecutorService; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.Executor; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.reactivestreams.Subscriber; -import org.reactivestreams.Subscription; - -import org.springframework.core.task.support.ExecutorServiceAdapter; -import org.springframework.integration.kafka.core.KafkaMessage; -import org.springframework.scheduling.concurrent.ConcurrentTaskExecutor; -import org.springframework.util.ObjectUtils; - -import reactor.core.processor.RingBufferProcessor; - -/** - * Invokes a delegate {@link MessageListener} for all the messages passed to it, storing - * them in an internal queue. - * - * @author Marius Bogoevici - * @author Stephane Maldini - */ -class QueueingMessageListenerInvoker { - - private static Log log = LogFactory.getLog(QueueingMessageListenerInvoker.class); - - private final MessageListener messageListener; - - private final AcknowledgingMessageListener acknowledgingMessageListener; - - private final OffsetManager offsetManager; - - private final ErrorHandler errorHandler; - - private final int capacity; - - private final boolean autoCommitOnError; - - private final ExecutorService executorService; - - private volatile RingBufferProcessor ringBufferProcessor; - - private volatile CancelableSingleTaskExecutorService cancelableExecutorService; - - private volatile boolean running = false; - - private volatile CountDownLatch shutdownLatch; - - public QueueingMessageListenerInvoker(int capacity, final OffsetManager offsetManager, Object delegate, - final ErrorHandler errorHandler, Executor executor, boolean autoCommitOnError) { - this.capacity = capacity; - this.autoCommitOnError = autoCommitOnError; - if (delegate instanceof MessageListener) { - this.messageListener = (MessageListener) delegate; - this.acknowledgingMessageListener = null; - } - else if (delegate instanceof AcknowledgingMessageListener) { - this.acknowledgingMessageListener = (AcknowledgingMessageListener) delegate; - this.messageListener = null; - } - else { - // it's neither, an exception will be thrown - throw new IllegalArgumentException("Either a " - + MessageListener.class.getName() + " or a " - + AcknowledgingMessageListener.class.getName() + " must be provided"); - } - this.offsetManager = offsetManager; - this.errorHandler = errorHandler; - if (executor != null) { - this.executorService = new ExecutorServiceAdapter(new ConcurrentTaskExecutor( - executor)); - } - else { - this.executorService = Executors.newSingleThreadExecutor(); - } - } - - /** - * Add a message to the queue, waiting if the queue has reached its maximum capacity. - * - * @param message the KafkaMessage to add - */ - public void enqueue(KafkaMessage message) { - if (this.running) { - ringBufferProcessor.onNext(message); - } - } - - public synchronized void start() { - if (!this.running) { - this.running = true; - // wraps the executor, allowing for the interruption of the processing thread - // on shutdown without - // stopping the executor service (which may be injected) - cancelableExecutorService = new CancelableSingleTaskExecutorService(executorService); - this.ringBufferProcessor = RingBufferProcessor.share(cancelableExecutorService, capacity); - this.ringBufferProcessor.subscribe(new KafkaMessageDispatchingSubscriber()); - } - } - - public synchronized void stop(long stopTimeout) { - if (this.running) { - this.running = false; - if (ringBufferProcessor != null) { - // cancel the current task - shutdownLatch = new CountDownLatch(1); - cancelableExecutorService.cancelTask(); - // allow the processor to complete, even if no more messages will be processed - ringBufferProcessor.onComplete(); - ringBufferProcessor = null; - try { - shutdownLatch.await(stopTimeout, TimeUnit.MILLISECONDS); - } - catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - finally { - shutdownLatch = null; - } - } - } - } - - /** - * {@link ExecutorService} implementation that supports the execution of a single - * task, deferring to the wrapped instance. Allows for interrupting the - * {@link RingBufferProcessor}'s executing thread, in case it is blocking. - * @since 1.3 - */ - private static class CancelableSingleTaskExecutorService extends - AbstractExecutorService { - - private Future submittedTask; - - private final ExecutorService executor; - - public CancelableSingleTaskExecutorService(ExecutorService executor) { - this.executor = executor; - } - - @Override - public void execute(Runnable task) { - if (submittedTask == null) { - submittedTask = this.executor.submit(task); - } - else { - throw new IllegalArgumentException("Cannot submit more than one task"); - } - } - - @Override - public void shutdown() { - throw new IllegalStateException("Manual shutdown not supported"); - } - - @Override - public List shutdownNow() { - throw new IllegalStateException("Manual shutdown not supported"); - } - - @Override - public boolean isShutdown() { - return false; - } - - @Override - public boolean isTerminated() { - return false; - } - - @Override - public boolean awaitTermination(long timeout, TimeUnit unit) - throws InterruptedException { - throw new IllegalStateException("Manual termination not supported"); - } - - private void cancelTask() { - if (submittedTask != null) { - submittedTask.cancel(true); - submittedTask = null; - } - } - } - - private class KafkaMessageDispatchingSubscriber implements Subscriber { - - @Override - public void onSubscribe(Subscription s) { - s.request(Long.MAX_VALUE); - } - - @Override - public void onNext(KafkaMessage kafkaMessage) { - if (running) { - try { - if (messageListener != null) { - messageListener.onMessage(kafkaMessage); - offsetManager.updateOffset(kafkaMessage.getMetadata() - .getPartition(), kafkaMessage.getMetadata() - .getNextOffset()); - } - else { - acknowledgingMessageListener.onMessage(kafkaMessage, - new DefaultAcknowledgment(offsetManager, kafkaMessage)); - } - } - catch (Exception e) { - // we handle errors here so that we make sure that offsets are handled - // concurrently - if (errorHandler != null) { - errorHandler.handle(e, kafkaMessage); - if (autoCommitOnError) { - offsetManager.updateOffset(kafkaMessage.getMetadata() - .getPartition(), kafkaMessage.getMetadata() - .getNextOffset()); - } - } - } - } - else { - if (log.isDebugEnabled()) { - log.debug("Message discarded on shutdown (no offsets have been committed): " - + ObjectUtils.nullSafeToString(kafkaMessage)); - } - } - } - - @Override - public void onError(Throwable t) { - // ignore - } - - @Override - public void onComplete() { - if (shutdownLatch != null) { - shutdownLatch.countDown(); - } - } - } -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/WindowingOffsetManager.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/WindowingOffsetManager.java deleted file mode 100644 index 5cf33a98ff..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/WindowingOffsetManager.java +++ /dev/null @@ -1,310 +0,0 @@ -/* - * Copyright 2016 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.listener; - -import java.io.IOException; -import java.util.Collection; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; - -import org.springframework.beans.factory.DisposableBean; -import org.springframework.beans.factory.InitializingBean; -import org.springframework.integration.kafka.core.Partition; -import org.springframework.util.Assert; - -import reactor.Environment; -import reactor.core.processor.RingBufferProcessor; -import reactor.fn.BiFunction; -import reactor.fn.Consumer; -import reactor.fn.Function; -import reactor.rx.Stream; -import reactor.rx.Streams; -import reactor.rx.stream.GroupedStream; - -/** - * An {@link OffsetManager} that aggregates writes over a time or count window, using an underlying delegate to - * do the actual operations. Its purpose is to reduce the performance impact of writing operations - * wherever this is desirable. - *

- * A time window or a number of writes can be specified, or both. - * Defaults to 10 seconds window with {@link Integer#MAX_VALUE} buffer. - * @author Marius Bogoevici - * @author Artem Bilan - * @since 1.3.1 - */ -public class WindowingOffsetManager implements OffsetManager, InitializingBean, DisposableBean { - - static { - Environment.initializeIfEmpty(); - } - - private static final BiFunction maxFunction = new BiFunction() { - - @Override - public Long apply(Long aLong, Long bLong) { - return Math.max(aLong, bLong); - } - - }; - - private static final Function offsetFunction - = new Function() { - - @Override - public Long apply(PartitionAndOffset partitionAndOffset) { - return partitionAndOffset.getOffset(); - } - - }; - - private static final ComputeMaximumOffsetByPartitionFunction findHighestOffsetInPartitionGroup - = new ComputeMaximumOffsetByPartitionFunction(); - - private static final Function getPartitionFunction - = new Function() { - - @Override - public Partition apply(PartitionAndOffset partitionAndOffset) { - return partitionAndOffset.getPartition(); - } - - }; - - private static final FindHighestOffsetsByPartitionFunction findHighestOffsetsByPartition - = new FindHighestOffsetsByPartitionFunction(); - - private final Consumer delegateUpdateOffset = new Consumer() { - - @Override - public void accept(PartitionAndOffset partitionAndOffset) { - delegate.updateOffset(partitionAndOffset.getPartition(), partitionAndOffset.getOffset()); - } - - }; - - private final Consumer offsetComplete = new Consumer() { - - @Override - public void accept(Void aVoid) { - createOffsetsStream(); - } - - }; - - private final ReadWriteLock offsetsLock = new ReentrantReadWriteLock(); - - private final OffsetManager delegate; - - private long timespan = 10 * 1000; - - private int count = Integer.MAX_VALUE; - - private int shutdownTimeout = 2000; - - private volatile RingBufferProcessor offsets; - - private volatile boolean closed; - - public WindowingOffsetManager(OffsetManager offsetManager) { - this.delegate = offsetManager; - } - - /** - * The timespan for aggregating write operations, before invoking the underlying {@link OffsetManager}. - * @param timespan duration in milliseconds - */ - public void setTimespan(long timespan) { - Assert.isTrue(timespan >= 0, "Timespan must be a positive value"); - this.timespan = timespan; - } - - /** - * How many writes should be aggregated, before invoking the underlying {@link OffsetManager}. Setting this value - * to 1 effectively disables windowing. - * @param count number of writes - */ - public void setCount(int count) { - Assert.isTrue(count >= 0, "Count must be a positive value"); - this.count = count; - } - - /** - * The timeout that {@link #close()} and {@link #destroy()} - * operations will wait for receiving a confirmation that - * the underlying writes have been processed. - * @param shutdownTimeout duration in milliseconds - */ - public void setShutdownTimeout(int shutdownTimeout) { - this.shutdownTimeout = shutdownTimeout; - } - - @Override - public void afterPropertiesSet() throws Exception { - if (this.count != 1) { - createOffsetsStream(); - } - } - - private void createOffsetsStream() { - if (!this.closed) { - this.offsetsLock.writeLock().lock(); - try { - this.offsets = RingBufferProcessor.share("spring-integration-kafka-offset", 1024); - } - finally { - this.offsetsLock.writeLock().unlock(); - } - Streams.wrap(this.offsets) - .window(this.count, timespan, TimeUnit.MILLISECONDS) - .flatMap(findHighestOffsetsByPartition) - .consume(this.delegateUpdateOffset, null, this.offsetComplete); - - } - } - - @Override - public void destroy() throws Exception { - flush(); - close(); - if (this.delegate instanceof DisposableBean) { - ((DisposableBean) this.delegate).destroy(); - } - } - - @Override - public void updateOffset(Partition partition, long offset) { - if (this.offsets != null) { - this.offsetsLock.readLock().lock(); - try { - this.offsets.onNext(new PartitionAndOffset(partition, offset)); - } - finally { - this.offsetsLock.readLock().unlock(); - } - } - else { - this.delegate.updateOffset(partition, offset); - } - } - - @Override - public long getOffset(Partition partition) { - doFlush(); - return this.delegate.getOffset(partition); - } - - @Override - public void deleteOffset(Partition partition) { - doFlush(); - this.delegate.deleteOffset(partition); - } - - @Override - public void resetOffsets(Collection partition) { - doFlush(); - this.delegate.resetOffsets(partition); - } - - @Override - public void close() throws IOException { - this.closed = true; - this.delegate.close(); - } - - @Override - public void flush() throws IOException { - if (this.offsets != null) { - this.offsets.awaitAndShutdown(this.shutdownTimeout, TimeUnit.MILLISECONDS); - } - this.delegate.flush(); - } - - private void doFlush() { - try { - flush(); - } - catch (IOException e) { - throw new IllegalStateException(e); - } - } - - - private static class PartitionAndOffset { - - private final Partition partition; - - private final Long offset; - - public PartitionAndOffset(Partition partition, Long offset) { - this.partition = partition; - this.offset = offset; - } - - public Partition getPartition() { - return partition; - } - - public Long getOffset() { - return offset; - } - - @Override - public String toString() { - return "PartitionAndOffset{" + - "partition=" + partition + - ", offset=" + offset + - '}'; - } - - } - - - private static class ComputeMaximumOffsetByPartitionFunction - implements Function, Stream> { - - @Override - public Stream apply(final GroupedStream group) { - return group - .map(offsetFunction) - .reduce(maxFunction) - .map(new Function() { - - @Override - public PartitionAndOffset apply(Long offset) { - return new PartitionAndOffset(group.key(), offset); - } - - }); - } - - } - - private static class FindHighestOffsetsByPartitionFunction - implements Function, Stream> { - - @Override - public Stream apply(Stream windowBuffer) { - return windowBuffer - .groupBy(getPartitionFunction) - .flatMap(findHighestOffsetInPartitionGroup); - } - - } - -} - diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/package-info.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/package-info.java deleted file mode 100644 index c2a49222f5..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/listener/package-info.java +++ /dev/null @@ -1,4 +0,0 @@ -/** - * Provides message listener container support - */ -package org.springframework.integration.kafka.listener; diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/outbound/KafkaProducerMessageHandler.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/outbound/KafkaProducerMessageHandler.java index bca32f3a5c..ee5fa64e6d 100644 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/outbound/KafkaProducerMessageHandler.java +++ b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/outbound/KafkaProducerMessageHandler.java @@ -20,9 +20,10 @@ import org.springframework.expression.EvaluationContext; import org.springframework.expression.Expression; import org.springframework.integration.expression.ExpressionUtils; import org.springframework.integration.handler.AbstractMessageHandler; -import org.springframework.integration.kafka.support.KafkaHeaders; -import org.springframework.integration.kafka.support.KafkaProducerContext; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.support.KafkaHeaders; import org.springframework.messaging.Message; +import org.springframework.util.Assert; /** * @author Soby Chacko @@ -31,9 +32,9 @@ import org.springframework.messaging.Message; * @author Marius Bogoevici * @since 0.5 */ -public class KafkaProducerMessageHandler extends AbstractMessageHandler { +public class KafkaProducerMessageHandler extends AbstractMessageHandler { - private final KafkaProducerContext kafkaProducerContext; + private final KafkaTemplate kafkaTemplate; private EvaluationContext evaluationContext; @@ -45,8 +46,9 @@ public class KafkaProducerMessageHandler extends AbstractMessageHandler { private volatile Expression partitionIdExpression; - public KafkaProducerMessageHandler(final KafkaProducerContext kafkaProducerContext) { - this.kafkaProducerContext = kafkaProducerContext; + public KafkaProducerMessageHandler(final KafkaTemplate kafkaTemplate) { + Assert.notNull(kafkaTemplate, "kafkaTemplate cannot be null"); + this.kafkaTemplate = kafkaTemplate; } /** @@ -83,8 +85,8 @@ public class KafkaProducerMessageHandler extends AbstractMessageHandler { this.partitionIdExpression = partitionIdExpression; } - public KafkaProducerContext getKafkaProducerContext() { - return this.kafkaProducerContext; + public KafkaTemplate getKafkaTemplate() { + return this.kafkaTemplate; } @Override @@ -93,6 +95,7 @@ public class KafkaProducerMessageHandler extends AbstractMessageHandler { this.evaluationContext = ExpressionUtils.createStandardEvaluationContext(getBeanFactory()); } + @SuppressWarnings("unchecked") @Override protected void handleMessageInternal(final Message message) throws Exception { String topic = this.topicExpression != null ? @@ -108,7 +111,13 @@ public class KafkaProducerMessageHandler extends AbstractMessageHandler { ? this.messageKeyExpression.getValue(this.evaluationContext, message) : message.getHeaders().get(KafkaHeaders.MESSAGE_KEY); - this.kafkaProducerContext.send(topic, partitionId, messageKey, message.getPayload()); + // TODO: Add KafkaTemplate method with topic, partition, data only (no key) + if (partitionId == null) { + this.kafkaTemplate.convertAndSend(topic, (K) messageKey, ((V) message.getPayload())); + } + else { + this.kafkaTemplate.convertAndSend(topic, partitionId, (K) messageKey, ((V) message.getPayload())); + } } @Override diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/avro/AvroDatumSupport.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/avro/AvroDatumSupport.java deleted file mode 100644 index 808c5a0bce..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/avro/AvroDatumSupport.java +++ /dev/null @@ -1,41 +0,0 @@ -package org.springframework.integration.kafka.serializer.avro; - -import org.apache.avro.io.DatumReader; -import org.apache.avro.io.DatumWriter; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import java.io.IOException; - -/** - * @author Soby Chacko - * @since 0.5 - */ -public abstract class AvroDatumSupport { - - private static final Log LOG = LogFactory.getLog(AvroDatumSupport.class); - - private final AvroSerializer avroSerializer; - - protected AvroDatumSupport() { - this.avroSerializer = new AvroSerializer(); - } - - public byte[] toBytes(final T source, final DatumWriter writer) { - try { - return avroSerializer.serialize(source, writer); - } catch (IOException e) { - LOG.error("Failed to encode source: " + e); - } - return null; - } - - public T fromBytes(final byte[] bytes, final DatumReader reader) { - try { - return avroSerializer.deserialize(bytes, reader); - } catch (IOException e) { - LOG.error("Failed to decode byte array: " + e); - } - return null; - } -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/avro/AvroReflectDatumBackedKafkaDecoder.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/avro/AvroReflectDatumBackedKafkaDecoder.java deleted file mode 100644 index 68a6663cca..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/avro/AvroReflectDatumBackedKafkaDecoder.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.kafka.serializer.avro; - -import kafka.serializer.Decoder; - -import org.apache.avro.io.DatumReader; -import org.apache.avro.reflect.ReflectDatumReader; - -/** - * @author Soby Chacko - * @since 0.5 - */ -public class AvroReflectDatumBackedKafkaDecoder extends AvroDatumSupport implements Decoder { - - private final DatumReader reader; - - public AvroReflectDatumBackedKafkaDecoder(final Class clazz) { - this.reader = new ReflectDatumReader(clazz); - } - - @Override - public T fromBytes(final byte[] bytes) { - return fromBytes(bytes, reader); - } -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/avro/AvroReflectDatumBackedKafkaEncoder.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/avro/AvroReflectDatumBackedKafkaEncoder.java deleted file mode 100644 index 4d51d096d7..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/avro/AvroReflectDatumBackedKafkaEncoder.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.kafka.serializer.avro; - -import kafka.serializer.Encoder; - -import org.apache.avro.io.DatumWriter; -import org.apache.avro.reflect.ReflectDatumWriter; - -/** - * @author Soby Chacko - * @since 0.5 - */ -public class AvroReflectDatumBackedKafkaEncoder extends AvroDatumSupport implements Encoder { - - private final DatumWriter writer; - - public AvroReflectDatumBackedKafkaEncoder(final Class clazz) { - this.writer = new ReflectDatumWriter(clazz); - } - - @Override - public byte[] toBytes(final T source) { - return toBytes(source, writer); - } -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/avro/AvroSerializer.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/avro/AvroSerializer.java deleted file mode 100644 index 2a98894c58..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/avro/AvroSerializer.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.kafka.serializer.avro; - -import org.apache.avro.io.DatumReader; -import org.apache.avro.io.DatumWriter; -import org.apache.avro.io.Decoder; -import org.apache.avro.io.DecoderFactory; -import org.apache.avro.io.Encoder; -import org.apache.avro.io.EncoderFactory; - -import java.io.ByteArrayOutputStream; -import java.io.IOException; - -/** - * @author Soby Chacko - * @since 0.5 - */ -public class AvroSerializer { - - public T deserialize(final byte[] bytes, final DatumReader reader) throws IOException { - final Decoder decoder = DecoderFactory.get().binaryDecoder(bytes, null); - return reader.read(null, decoder); - } - - public byte[] serialize(final T input, final DatumWriter writer) throws IOException { - final ByteArrayOutputStream stream = new ByteArrayOutputStream(); - - final Encoder encoder = EncoderFactory.get().binaryEncoder(stream, null); - writer.write(input, encoder); - encoder.flush(); - - return stream.toByteArray(); - } -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/avro/AvroSpecificDatumBackedKafkaDecoder.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/avro/AvroSpecificDatumBackedKafkaDecoder.java deleted file mode 100644 index 9ad7ee6b54..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/avro/AvroSpecificDatumBackedKafkaDecoder.java +++ /dev/null @@ -1,24 +0,0 @@ -package org.springframework.integration.kafka.serializer.avro; - -import kafka.serializer.Decoder; - -import org.apache.avro.io.DatumReader; -import org.apache.avro.specific.SpecificDatumReader; - -/** - * @author Soby Chacko - * @since 0.5 - */ -public class AvroSpecificDatumBackedKafkaDecoder extends AvroDatumSupport implements Decoder { - - private final DatumReader reader; - - public AvroSpecificDatumBackedKafkaDecoder(final Class specificRecordBase) { - this.reader = new SpecificDatumReader(specificRecordBase); - } - - @Override - public T fromBytes(final byte[] bytes) { - return fromBytes(bytes, reader); - } -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/avro/AvroSpecificDatumBackedKafkaEncoder.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/avro/AvroSpecificDatumBackedKafkaEncoder.java deleted file mode 100644 index 92fb4acd21..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/avro/AvroSpecificDatumBackedKafkaEncoder.java +++ /dev/null @@ -1,24 +0,0 @@ -package org.springframework.integration.kafka.serializer.avro; - -import kafka.serializer.Encoder; - -import org.apache.avro.io.DatumWriter; -import org.apache.avro.specific.SpecificDatumWriter; - -/** - * @author Soby Chacko - * @since 0.5 - */ -public class AvroSpecificDatumBackedKafkaEncoder extends AvroDatumSupport implements Encoder { - - private final DatumWriter writer; - - public AvroSpecificDatumBackedKafkaEncoder(final Class specificRecordClazz) { - this.writer = new SpecificDatumWriter(specificRecordClazz); - } - - @Override - public byte[] toBytes(final T source) { - return toBytes(source, writer); - } -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/common/StringDecoder.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/common/StringDecoder.java deleted file mode 100644 index d5b3e200a3..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/common/StringDecoder.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright 2002-2014 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.kafka.serializer.common; - -import java.util.Properties; - -import kafka.serializer.Decoder; -import kafka.utils.VerifiableProperties; - - -/** - * String Decoder for Kafka message key/value decoding. - * The Default decoder returns the same byte array it takes in. - * - * @author Soby Chacko - * @author Ilayaperumal Gopinathan - */ -public class StringDecoder implements Decoder { - - private final kafka.serializer.StringDecoder stringDecoder; - - public StringDecoder() { - this("UTF8"); - } - - public StringDecoder(final String encoding) { - final Properties props = new Properties(); - props.put("serializer.encoding", encoding); - this.stringDecoder = new kafka.serializer.StringDecoder(new VerifiableProperties(props)); - } - - @Override - public String fromBytes(byte[] bytes) { - return this.stringDecoder.fromBytes(bytes); - } - -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/common/StringEncoder.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/common/StringEncoder.java deleted file mode 100644 index 2382ea58f1..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/serializer/common/StringEncoder.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.kafka.serializer.common; - -import java.util.Properties; - -import kafka.serializer.Encoder; -import kafka.utils.VerifiableProperties; - -/** - * @author Soby Chacko - * @author Marius Bogoevici - * @since 0.5 - */ -public class StringEncoder implements Encoder { - - private kafka.serializer.StringEncoder stringEncoder; - - public StringEncoder() { - this("UTF-8"); - } - - public StringEncoder(String encoding) { - final Properties props = new Properties(); - props.put("serializer.encoding", encoding); - final VerifiableProperties verifiableProperties = new VerifiableProperties(props); - stringEncoder = new kafka.serializer.StringEncoder(verifiableProperties); - } - - @Override - public byte[] toBytes(final String value) { - return stringEncoder.toBytes(value); - } -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ConsumerConfigFactoryBean.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ConsumerConfigFactoryBean.java deleted file mode 100644 index 8fda57e6ed..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ConsumerConfigFactoryBean.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.kafka.support; - -import java.util.Properties; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import org.springframework.beans.factory.FactoryBean; - -import kafka.consumer.ConsumerConfig; - -/** - * @author Soby Chacko - * @since 0.5 - */ -@Deprecated -@SuppressWarnings("deprecation") -public class ConsumerConfigFactoryBean implements FactoryBean { - - private static final Log LOGGER = LogFactory.getLog(ConsumerConfigFactoryBean.class); - - private final ConsumerMetadata consumerMetadata; - private final ZookeeperConnect zookeeperConnect; - private Properties consumerProperties = new Properties(); - - public ConsumerConfigFactoryBean(final ConsumerMetadata consumerMetadata, - final ZookeeperConnect zookeeperConnect, final Properties consumerProperties) { - this.consumerMetadata = consumerMetadata; - this.zookeeperConnect = zookeeperConnect; - if (consumerProperties != null) { - this.consumerProperties = consumerProperties; - } - } - - public ConsumerConfigFactoryBean(final ConsumerMetadata consumerMetadata, - final ZookeeperConnect zookeeperConnect) { - this(consumerMetadata, zookeeperConnect, null); - } - - @Override - public ConsumerConfig getObject() throws Exception { - final Properties properties = new Properties(); - properties.putAll(consumerProperties); - properties.put("zookeeper.connect", zookeeperConnect.getZkConnect()); - properties.put("zookeeper.session.timeout.ms", zookeeperConnect.getZkSessionTimeout()); - properties.put("zookeeper.sync.time.ms", zookeeperConnect.getZkSyncTime()); - - // Overriding the default value of -1, which will make the consumer to - // wait indefinitely - if (!properties.containsKey("consumer.timeout.ms")) { - properties.put("consumer.timeout.ms", consumerMetadata.getConsumerTimeout()); - } - - properties.put("group.id", consumerMetadata.getGroupId()); - - LOGGER.info("Using consumer properties => " + properties); - - return new ConsumerConfig(properties); - } - - @Override - public Class getObjectType() { - return ConsumerConfig.class; - } - - @Override - public boolean isSingleton() { - return true; - } -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ConsumerConfiguration.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ConsumerConfiguration.java deleted file mode 100644 index ba9333cc84..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ConsumerConfiguration.java +++ /dev/null @@ -1,280 +0,0 @@ -/* - * Copyright 2002-2015 the original author or authors. Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. You may obtain a copy of the License at - * http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, - * either express or implied. See the License for the specific language governing permissions and limitations under the - * License. - */ - -package org.springframework.integration.kafka.support; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.LinkedList; -import java.util.List; -import java.util.Map; -import java.util.concurrent.Callable; -import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.Executor; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.Future; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import org.springframework.messaging.MessagingException; -import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; -import org.springframework.util.Assert; - -import kafka.consumer.ConsumerTimeoutException; -import kafka.consumer.KafkaStream; -import kafka.javaapi.consumer.ConsumerConnector; -import kafka.message.MessageAndMetadata; - -/** - * @author Soby Chacko - * @author Rajasekar Elango - * @author Artem Bilan - * @since 0.5 - */ -@Deprecated -@SuppressWarnings("deprecation") -public class ConsumerConfiguration { - - private static final Log LOGGER = LogFactory.getLog(ConsumerConfiguration.class); - - private final ConsumerMetadata consumerMetadata; - - private final ConsumerConnectionProvider consumerConnectionProvider; - - private final MessageLeftOverTracker messageLeftOverTracker; - - private ConsumerConnector consumerConnector; - - private volatile int count = 0; - - private int maxMessages = 1; - - private Collection>> consumerMessageStreams; - - private ExecutorService executorService = Executors.newCachedThreadPool(); - - private boolean executorExplicitlySet; - - private volatile boolean stopped; - - public ConsumerConfiguration(final ConsumerMetadata consumerMetadata, - final ConsumerConnectionProvider consumerConnectionProvider, - final MessageLeftOverTracker messageLeftOverTracker) { - this.consumerMetadata = consumerMetadata; - this.consumerConnectionProvider = consumerConnectionProvider; - this.messageLeftOverTracker = messageLeftOverTracker; - } - - public void setExecutor(Executor executor) { - boolean isExecutorService = executor instanceof ExecutorService; - boolean isThreadPoolTaskExecutor = executor instanceof ThreadPoolTaskExecutor; - Assert.isTrue(isExecutorService || isThreadPoolTaskExecutor); - if (isExecutorService) { - this.executorService = (ExecutorService) executor; - } - else { - this.executorService = ((ThreadPoolTaskExecutor) executor).getThreadPoolExecutor(); - } - this.executorExplicitlySet = true; - } - - public ConsumerMetadata getConsumerMetadata() { - return consumerMetadata; - } - - public Map>> receive() { - count = messageLeftOverTracker.getCurrentCount(); - final Object lock = new Object(); - - final List>>> tasks = new LinkedList>>>(); - - for (final List> streams : createConsumerMessageStreams()) { - for (final KafkaStream stream : streams) { - tasks.add(new Callable>>() { - @Override - public List> call() throws Exception { - final List> rawMessages = new ArrayList>(); - try { - while (count < maxMessages) { - final MessageAndMetadata messageAndMetadata = stream.iterator().next(); - synchronized (lock) { - if (count < maxMessages) { - rawMessages.add(messageAndMetadata); - count++; - } - else { - messageLeftOverTracker.addMessageAndMetadata(messageAndMetadata); - } - } - } - } - catch (ConsumerTimeoutException cte) { - LOGGER.debug("Consumer timed out"); - } - return rawMessages; - } - }); - } - } - return executeTasks(tasks); - } - - private Map>> executeTasks( - final List>>> tasks) { - - final Map>> messages = new ConcurrentHashMap>>(); - messages.putAll(getLeftOverMessageMap()); - - try { - for (final Future>> result : this.executorService.invokeAll(tasks)) { - if (!result.get().isEmpty()) { - final String topic = result.get().get(0).topic(); - if (!messages.containsKey(topic)) { - messages.put(topic, getPayload(result.get())); - } - else { - - final Map> existingPayloadMap = messages.get(topic); - getPayload(result.get(), existingPayloadMap); - } - } - } - } - catch (Exception e) { - if (!this.stopped) { - throw new MessagingException("Consuming from Kafka failed", e); - } - else { - LOGGER.warn("Consuming from Kafka failed", e); - } - } - - if (messages.isEmpty()) { - return null; - } - - return messages; - } - - private Map>> getLeftOverMessageMap() { - - final Map>> messages = new ConcurrentHashMap>>(); - - for (final MessageAndMetadata mamd : messageLeftOverTracker.getMessageLeftOverFromPreviousPoll()) { - final String topic = mamd.topic(); - - if (!messages.containsKey(topic)) { - final List> l = new ArrayList>(); - l.add(mamd); - messages.put(topic, getPayload(l)); - } - else { - final Map> existingPayloadMap = messages.get(topic); - final List> l = new ArrayList>(); - l.add(mamd); - getPayload(l, existingPayloadMap); - } - } - messageLeftOverTracker.clearMessagesLeftOver(); - return messages; - } - - private Map> getPayload(final List> messageAndMetadatas) { - final Map> payloadMap = new ConcurrentHashMap>(); - - for (final MessageAndMetadata messageAndMetadata : messageAndMetadatas) { - if (!payloadMap.containsKey(messageAndMetadata.partition())) { - final List payload = new ArrayList(); - payload.add(messageAndMetadata.message()); - payloadMap.put(messageAndMetadata.partition(), payload); - } - else { - final List payload = payloadMap.get(messageAndMetadata.partition()); - payload.add(messageAndMetadata.message()); - } - - } - - return payloadMap; - } - - private void getPayload(final List> messageAndMetadatas, - final Map> existingPayloadMap) { - for (final MessageAndMetadata messageAndMetadata : messageAndMetadatas) { - if (!existingPayloadMap.containsKey(messageAndMetadata.partition())) { - final List payload = new ArrayList(); - payload.add(messageAndMetadata.message()); - existingPayloadMap.put(messageAndMetadata.partition(), payload); - } - else { - final List payload = existingPayloadMap.get(messageAndMetadata.partition()); - payload.add(messageAndMetadata.message()); - } - } - } - - private Collection>> createConsumerMessageStreams() { - if (consumerMessageStreams == null) { - if (!(consumerMetadata.getTopicStreamMap() == null || consumerMetadata.getTopicStreamMap().isEmpty())) { - consumerMessageStreams = createMessageStreamsForTopic().values(); - } - else { - consumerMessageStreams = new ArrayList>>(); - consumerMessageStreams.add(createMessageStreamsForTopicFilter()); - } - } - return consumerMessageStreams; - } - - public Map>> createMessageStreamsForTopic() { - return getConsumerConnector().createMessageStreams(consumerMetadata.getTopicStreamMap(), - consumerMetadata.getKeyDecoder(), consumerMetadata.getValueDecoder()); - } - - public List> createMessageStreamsForTopicFilter() { - List> messageStream = new ArrayList>(); - TopicFilterConfiguration topicFilterConfiguration = consumerMetadata.getTopicFilterConfiguration(); - if (topicFilterConfiguration != null) { - messageStream = getConsumerConnector().createMessageStreamsByFilter( - topicFilterConfiguration.getTopicFilter(), topicFilterConfiguration.getNumberOfStreams(), - consumerMetadata.getKeyDecoder(), consumerMetadata.getValueDecoder()); - } - else { - LOGGER.warn("No Topic Filter Configuration defined"); - } - - return messageStream; - } - - public int getMaxMessages() { - return maxMessages; - } - - public void setMaxMessages(final int maxMessages) { - this.maxMessages = maxMessages; - } - - public ConsumerConnector getConsumerConnector() { - if (consumerConnector == null) { - consumerConnector = consumerConnectionProvider.getConsumerConnector(); - } - return consumerConnector; - } - - public void shutdown() { - this.stopped = true; - if (!this.executorExplicitlySet) { - this.executorService.shutdownNow(); - } - getConsumerConnector().shutdown(); - } - -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ConsumerConnectionProvider.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ConsumerConnectionProvider.java deleted file mode 100644 index 539596bff4..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ConsumerConnectionProvider.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.kafka.support; - -import kafka.consumer.ConsumerConfig; -import kafka.javaapi.consumer.ConsumerConnector; - -/** - * @author Soby Chacko - * @since 0.5 - */ -@Deprecated -public class ConsumerConnectionProvider { - - private final ConsumerConfig consumerConfig; - - public ConsumerConnectionProvider(final ConsumerConfig consumerConfig) { - this.consumerConfig = consumerConfig; - } - - public ConsumerConnector getConsumerConnector() { - return kafka.consumer.Consumer.createJavaConsumerConnector(consumerConfig); - } -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ConsumerMetadata.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ConsumerMetadata.java deleted file mode 100644 index 3428f8aea2..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ConsumerMetadata.java +++ /dev/null @@ -1,203 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.kafka.support; - -import java.util.Map; - -import org.springframework.beans.factory.InitializingBean; -import org.springframework.integration.kafka.core.KafkaConsumerDefaults; - -import kafka.serializer.Decoder; -import kafka.serializer.DefaultDecoder; - -/** - * @author Soby Chacko - * @author Rajasekar Elango - * @since 0.5 - */ -@Deprecated -public class ConsumerMetadata implements InitializingBean { - - //High level consumer defaults - private String groupId = KafkaConsumerDefaults.GROUP_ID; - private String socketTimeout = KafkaConsumerDefaults.SOCKET_TIMEOUT; - private String socketBufferSize = KafkaConsumerDefaults.SOCKET_BUFFER_SIZE; - private String fetchSize = KafkaConsumerDefaults.FETCH_SIZE; - private String backoffIncrement = KafkaConsumerDefaults.BACKOFF_INCREMENT; - private String queuedChunksMax = KafkaConsumerDefaults.QUEUED_CHUNKS_MAX; - private String autoCommitEnable = KafkaConsumerDefaults.AUTO_COMMIT_ENABLE; - private String autoCommitInterval = KafkaConsumerDefaults.AUTO_COMMIT_INTERVAL; - private String autoOffsetReset = KafkaConsumerDefaults.AUTO_OFFSET_RESET; - private String rebalanceRetriesMax = KafkaConsumerDefaults.REBALANCE_RETRIES_MAX; - private String consumerTimeout = KafkaConsumerDefaults.CONSUMER_TIMEOUT; - - private String topic; - private int streams; - private Decoder valueDecoder; - private Decoder keyDecoder; - private Map topicStreamMap; - private TopicFilterConfiguration topicFilterConfiguration; - - public String getGroupId() { - return groupId; - } - - public void setGroupId(final String groupId) { - this.groupId = groupId; - } - - public String getSocketTimeout() { - return socketTimeout; - } - - public void setSocketTimeout(final String socketTimeout) { - this.socketTimeout = socketTimeout; - } - - public String getSocketBufferSize() { - return socketBufferSize; - } - - public void setSocketBufferSize(final String socketBufferSize) { - this.socketBufferSize = socketBufferSize; - } - - public String getFetchSize() { - return fetchSize; - } - - public void setFetchSize(final String fetchSize) { - this.fetchSize = fetchSize; - } - - public String getBackoffIncrement() { - return backoffIncrement; - } - - public void setBackoffIncrement(final String backoffIncrement) { - this.backoffIncrement = backoffIncrement; - } - - public String getQueuedChunksMax() { - return queuedChunksMax; - } - - public void setQueuedChunksMax(final String queuedChunksMax) { - this.queuedChunksMax = queuedChunksMax; - } - - public String getAutoCommitEnable() { - return autoCommitEnable; - } - - public void setAutoCommitEnable(final String autoCommitEnable) { - this.autoCommitEnable = autoCommitEnable; - } - - public String getAutoCommitInterval() { - return autoCommitInterval; - } - - public void setAutoCommitInterval(final String autoCommitInterval) { - this.autoCommitInterval = autoCommitInterval; - } - - public String getAutoOffsetReset() { - return autoOffsetReset; - } - - public void setAutoOffsetReset(final String autoOffsetReset) { - this.autoOffsetReset = autoOffsetReset; - } - - public String getRebalanceRetriesMax() { - return rebalanceRetriesMax; - } - - public void setRebalanceRetriesMax(final String rebalanceRetriesMax) { - this.rebalanceRetriesMax = rebalanceRetriesMax; - } - - public String getConsumerTimeout() { - return consumerTimeout; - } - - public void setConsumerTimeout(final String consumerTimeout) { - this.consumerTimeout = consumerTimeout; - } - - public String getTopic() { - return topic; - } - - public void setTopic(final String topic) { - this.topic = topic; - } - - public int getStreams() { - return streams; - } - - public void setStreams(final int streams) { - this.streams = streams; - } - - public Decoder getValueDecoder() { - return valueDecoder; - } - - public void setValueDecoder(final Decoder valueDecoder) { - this.valueDecoder = valueDecoder; - } - - public Decoder getKeyDecoder() { - return keyDecoder; - } - - public void setKeyDecoder(final Decoder keyDecoder) { - this.keyDecoder = keyDecoder; - } - - public Map getTopicStreamMap() { - return topicStreamMap; - } - - public void setTopicStreamMap(final Map topicStreamMap) { - this.topicStreamMap = topicStreamMap; - } - - @Override - @SuppressWarnings("unchecked") - public void afterPropertiesSet() throws Exception { - if (valueDecoder == null) { - setValueDecoder((Decoder) new DefaultDecoder(null)); - } - - if (keyDecoder == null) { - setKeyDecoder((Decoder) getValueDecoder()); - } - } - - public TopicFilterConfiguration getTopicFilterConfiguration() { - return topicFilterConfiguration; - } - - public void setTopicFilterConfiguration( - TopicFilterConfiguration topicFilterConfiguration) { - this.topicFilterConfiguration = topicFilterConfiguration; - } - -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/DefaultPartitioner.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/DefaultPartitioner.java deleted file mode 100644 index c0730814e1..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/DefaultPartitioner.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.kafka.support; - -import kafka.producer.Partitioner; -import kafka.utils.Utils; - -/** - * @author Soby Chacko - * @since 0.5 - * - * This class is for internal use only and therefore is at default access level - */ -@Deprecated -class DefaultPartitioner implements Partitioner { - /** - * Uses the key to calculate a partition bucket id for routing - * the data to the appropriate broker partition - * @return an integer between 0 and numPartitions-1 - */ - @Override - public int partition(final Object key, final int numPartitions) { - return Utils.abs(key.hashCode()) % numPartitions; - } - -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/DefaultProducerListener.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/DefaultProducerListener.java deleted file mode 100644 index 184b54c515..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/DefaultProducerListener.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.support; - -import org.apache.kafka.clients.producer.RecordMetadata; - -/** - * No-op implementation of @link ProducerListener}, to be used as base class for other implementations. - * - * @author Marius Bogoevici - * @since 1.3 - */ -public class DefaultProducerListener implements ProducerListener { - - @Override - public void onSuccess(String topic, Integer partition, Object key, Object value, RecordMetadata recordMetadata) { - } - - @Override - public void onError(String topic, Integer partition, Object key, Object value, Exception exception) { - } -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/KafkaConsumerContext.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/KafkaConsumerContext.java deleted file mode 100644 index e938c1c272..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/KafkaConsumerContext.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright 2002-2014 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.support; - -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.springframework.beans.factory.DisposableBean; -import org.springframework.integration.kafka.core.KafkaConsumerDefaults; -import org.springframework.integration.support.MessageBuilder; -import org.springframework.messaging.Message; -import org.springframework.util.CollectionUtils; - -/** - * @author Soby Chacko - * @author Ilayaperumal Gopinathan - * @since 0.5 - */ -@Deprecated -@SuppressWarnings("deprecation") -public class KafkaConsumerContext implements DisposableBean { - - private Map> consumerConfigurations; - - private String consumerTimeout = KafkaConsumerDefaults.CONSUMER_TIMEOUT; - - private ZookeeperConnect zookeeperConnect; - - public String getConsumerTimeout() { - return this.consumerTimeout; - } - - public void setConsumerTimeout(final String consumerTimeout) { - this.consumerTimeout = consumerTimeout; - } - - public ZookeeperConnect getZookeeperConnect() { - return this.zookeeperConnect; - } - - public void setZookeeperConnect(final ZookeeperConnect zookeeperConnect) { - this.zookeeperConnect = zookeeperConnect; - } - - public void setConsumerConfigurations(Map> consumerConfigurations) { - this.consumerConfigurations = consumerConfigurations; - } - - public Map> getConsumerConfigurations() { - return this.consumerConfigurations; - } - - public ConsumerConfiguration getConsumerConfiguration(String groupId) { - return this.consumerConfigurations.get(groupId); - } - - public Message>>> receive() { - final Map>> consumedData = new HashMap>>(); - - for (final ConsumerConfiguration consumerConfiguration : getConsumerConfigurations().values()) { - final Map>> messages = consumerConfiguration.receive(); - - if (!CollectionUtils.isEmpty(messages)) { - consumedData.putAll(messages); - } - } - return consumedData.isEmpty() ? null : MessageBuilder.withPayload(consumedData).build(); - } - - @Override - public void destroy() throws Exception { - for (ConsumerConfiguration config : this.consumerConfigurations.values()) { - config.shutdown(); - } - } - -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/KafkaHeaders.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/KafkaHeaders.java deleted file mode 100644 index ace9e965e3..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/KafkaHeaders.java +++ /dev/null @@ -1,40 +0,0 @@ -/* - * Copyright 2014-2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.support; - -/** - * @author Artem Bilan - * @author Marius Bogoevici - * @since 1.0 - */ -public abstract class KafkaHeaders { - - private static final String PREFIX = "kafka_"; - - public static final String TOPIC = PREFIX + "topic"; - - public static final String MESSAGE_KEY = PREFIX + "messageKey"; - - public static final String PARTITION_ID = PREFIX + "partitionId"; - - public static final String OFFSET = PREFIX + "offset"; - - public static final String NEXT_OFFSET = PREFIX + "nextOffset"; - - public static final String ACKNOWLEDGMENT = PREFIX + "acknowledgment"; - -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/KafkaProducerContext.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/KafkaProducerContext.java deleted file mode 100644 index 836ca53c63..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/KafkaProducerContext.java +++ /dev/null @@ -1,215 +0,0 @@ -/* - * Copyright 2013-2016 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.support; - -import java.util.Collection; -import java.util.Map; -import java.util.concurrent.Future; -import java.util.concurrent.atomic.AtomicBoolean; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.kafka.clients.producer.RecordMetadata; - -import org.springframework.beans.factory.BeanNameAware; -import org.springframework.context.SmartLifecycle; -import org.springframework.integration.support.context.NamedComponent; -import org.springframework.util.StringUtils; - -/** - * @author Soby Chacko - * @author Rajasekar Elango - * @author Ilayaperumal Gopinathan - * @author Gary Russell - * @author Artem Bilan - * @author Marius Bogoevici - * @since 0.5 - */ -public class KafkaProducerContext implements SmartLifecycle, NamedComponent, BeanNameAware { - - private static final Log logger = LogFactory.getLog(KafkaProducerContext.class); - - private final AtomicBoolean running = new AtomicBoolean(); - - private volatile Map> producerConfigurations; - - private volatile ProducerConfiguration theProducerConfiguration; - - private String beanName; - - private int phase = 0; - - private boolean autoStartup = true; - - public ProducerConfiguration getTopicConfiguration(final String topic) { - if (this.theProducerConfiguration != null) { - if (topic.matches(this.theProducerConfiguration.getProducerMetadata().getTopic())) { - return this.theProducerConfiguration; - } - } - - Collection> topics = this.producerConfigurations.values(); - - for (final ProducerConfiguration producerConfiguration : topics) { - if (topic.matches(producerConfiguration.getProducerMetadata().getTopic())) { - return producerConfiguration; - } - } - return null; - } - - public Map> getProducerConfigurations() { - return this.producerConfigurations; - } - - public void setProducerConfigurations(Map> producerConfigurations) { - this.producerConfigurations = producerConfigurations; - if (this.producerConfigurations.size() == 1) { - this.theProducerConfiguration = this.producerConfigurations.values().iterator().next(); - } - } - - /** - * @return the component type. - * @since 1.0 - */ - @Override - public String getComponentType() { - return "kafka:producer-context"; - } - - /** - * @param name the bean name. - * @since 1.0 - */ - @Override - public void setBeanName(String name) { - this.beanName = name; - } - - /** - * @param phase the phase to set. - * @see SmartLifecycle - * @since 1.0 - */ - public void setPhase(int phase) { - this.phase = phase; - } - - /** - * @param autoStartup the autoStartup to set. - * @see SmartLifecycle - * @since 1.0 - */ - public void setAutoStartup(boolean autoStartup) { - this.autoStartup = autoStartup; - } - - /** - * @return the component name. - * @since 1.0 - */ - @Override - public String getComponentName() { - return this.beanName; - } - - protected void doStart() { - } - - protected void doStop() { - if (this.producerConfigurations != null) { - for (ProducerConfiguration producerConfiguration : this.producerConfigurations.values()) { - producerConfiguration.stop(); - } - } - } - - @Override - public final void start() { - if (this.running.compareAndSet(false, true)) { - doStart(); - } - else { - if (logger.isDebugEnabled()) { - logger.debug(getComponentType() + ":" + getComponentName() + " is already running"); - } - } - } - - @Override - public final void stop() { - if (this.running.compareAndSet(true, false)) { - doStop(); - } - else { - if (logger.isDebugEnabled()) { - logger.debug(getComponentType() + ":" + getComponentName() + " is not running"); - } - } - } - - @Override - public boolean isRunning() { - return this.running.get(); - } - - @Override - public int getPhase() { - return this.phase; - } - - @Override - public boolean isAutoStartup() { - return this.autoStartup; - } - - @Override - public void stop(Runnable callback) { - stop(); - callback.run(); - } - - public Future send(String topic, Object messageKey, Object messagePayload) { - return send(topic, null, messageKey, messagePayload); - } - - public Future send(String topic, Integer partition, Object messageKey, Object messagePayload) { - if (!this.running.get()) { - start(); - } - - // only try to look up for a producer configuration if the topic is passed as argument - // if no topic is configured, then we'll fall back to the default if a single - // producer configuration is available - ProducerConfiguration producerConfiguration = - StringUtils.hasText(topic) ? getTopicConfiguration(topic) : null; - - if (producerConfiguration != null) { - return producerConfiguration.convertAndSend(topic, partition, messageKey, messagePayload); - } - // if there is a single producer configuration then use that config to send message. - else if (this.theProducerConfiguration != null) { - return this.theProducerConfiguration.convertAndSend(topic, partition, messageKey, messagePayload); - } - else { - throw new IllegalStateException("Could not send messages as there are multiple producer configurations " + - "with no topic information found from the message header."); - } - } - -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/LoggingProducerListener.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/LoggingProducerListener.java deleted file mode 100644 index 4957b08d55..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/LoggingProducerListener.java +++ /dev/null @@ -1,83 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.support; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import org.springframework.util.ObjectUtils; - -/** - * {@link ProducerListener} that logs exceptions thrown when sending messages. - * - * @author Marius Bogoevici - * @since 1.3 - */ -public class LoggingProducerListener extends DefaultProducerListener { - - private static final Log log = LogFactory.getLog(LoggingProducerListener.class); - - private boolean includeContents = true; - - private int maxContentLogged = 100; - - /** - * Whether the log message should include the contents (key and payload). - * - * @param includeContents true if the contents of the message should be logged - */ - public void setIncludeContents(boolean includeContents) { - this.includeContents = includeContents; - } - - /** - * The maximum amount of data to be logged for either key or password. As message sizes may vary and - * become fairly large, this allows limiting the amount of data sent to logs. - * - * @param maxContentLogged the maximum amount of data being logged. - */ - public void setMaxContentLogged(int maxContentLogged) { - this.maxContentLogged = maxContentLogged; - } - - @Override - public void onError(String topic, Integer partition, Object key, Object payload, Exception exception) { - if (log.isErrorEnabled()) { - StringBuffer logOutput = new StringBuffer(); - logOutput.append("Exception thrown when sending a message"); - if (includeContents) { - logOutput.append(" with key='" - + toDisplayString(ObjectUtils.nullSafeToString(key), maxContentLogged) + "'"); - logOutput.append(" and payload='" - + toDisplayString(ObjectUtils.nullSafeToString(payload),maxContentLogged) + "'"); - } - logOutput.append(" to topic " + topic); - if (partition != null) { - logOutput.append(" and partition " +partition); - } - logOutput.append(":"); - log.error(logOutput, exception); - } - } - - private String toDisplayString(String original, int maxCharacters) { - if (original.length() <= maxCharacters) { - return original; - } - return original.substring(0, maxCharacters) + "..."; - } -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/MessageLeftOverTracker.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/MessageLeftOverTracker.java deleted file mode 100644 index 8021a0638f..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/MessageLeftOverTracker.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.kafka.support; - -import java.util.ArrayList; -import java.util.List; - -import kafka.message.MessageAndMetadata; - -/** - * @author Soby Chacko - * @since 0.5 - */ -@Deprecated -public class MessageLeftOverTracker { - private final List> messageLeftOverFromPreviousPoll = new ArrayList>(); - - public void addMessageAndMetadata(final MessageAndMetadata messageAndMetadata){ - messageLeftOverFromPreviousPoll.add(messageAndMetadata); - } - - public List> getMessageLeftOverFromPreviousPoll(){ - return messageLeftOverFromPreviousPoll; - } - - public void clearMessagesLeftOver(){ - messageLeftOverFromPreviousPoll.clear(); - } - - public int getCurrentCount() { - return messageLeftOverFromPreviousPoll.size(); - } -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ProducerConfiguration.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ProducerConfiguration.java deleted file mode 100644 index 65aab277ff..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ProducerConfiguration.java +++ /dev/null @@ -1,201 +0,0 @@ -/* - * Copyright 2013-2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.support; - -import java.util.HashSet; -import java.util.Set; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; -import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; - -import org.apache.kafka.clients.producer.Producer; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.clients.producer.RecordMetadata; -import org.apache.kafka.common.KafkaException; - -import org.springframework.core.convert.ConversionService; -import org.springframework.core.convert.TypeDescriptor; -import org.springframework.core.convert.converter.GenericConverter; -import org.springframework.core.convert.support.GenericConversionService; -import org.springframework.core.serializer.support.SerializingConverter; -import org.springframework.util.Assert; -import org.springframework.util.StringUtils; - -/** - * @author Soby Chacko - * @author Rajasekar Elango - * @author Ilayaperumal Gopinathan - * @author Gary Russell - * @author Marius Bogoevici - * @author Artem Bilan - * @author Martin Dam - * @since 0.5 - */ -public class ProducerConfiguration { - - private final Producer producer; - - private final ProducerMetadata producerMetadata; - - private ConversionService conversionService; - - private ProducerListener producerListener; - - public ProducerConfiguration(ProducerMetadata producerMetadata, Producer producer) { - Assert.notNull(producerMetadata); - Assert.notNull(producer); - this.producerMetadata = producerMetadata; - this.producer = producer; - GenericConversionService genericConversionService = new GenericConversionService(); - genericConversionService.addConverter(new StringBytesConverter()); - genericConversionService.addConverter(Object.class, byte[].class, new SerializingConverter()); - this.conversionService = genericConversionService; - } - - public void setConversionService(ConversionService conversionService) { - Assert.notNull(conversionService, "Conversion service must not be null"); - this.conversionService = conversionService; - } - - public void setProducerListener(ProducerListener producerListener) { - this.producerListener = producerListener; - } - - public ProducerMetadata getProducerMetadata() { - return this.producerMetadata; - } - - public Future send(String topic, K messageKey, V messagePayload) { - if (this.getProducerMetadata().getPartitioner() != null) { - String targetTopic = StringUtils.hasText(topic) ? topic : this.producerMetadata.getTopic(); - int partition = this.getProducerMetadata().getPartitioner().partition(messageKey, - this.producer.partitionsFor(targetTopic).size()); - return this.send(targetTopic, partition, messageKey, messagePayload); - } - return this.send(topic, null, messageKey, messagePayload); - } - - public Future send(String topic, Integer partition, K messageKey, V messagePayload) { - String targetTopic = StringUtils.hasText(topic) ? topic : this.producerMetadata.getTopic(); - //If partition is sent by producer context then that takes precedence over custom partitioner. - if (partition == null && this.getProducerMetadata().getPartitioner() != null) { - partition = this.getProducerMetadata().getPartitioner().partition(messageKey, - this.producer.partitionsFor(targetTopic).size()); - } - - ProducerRecord record = new ProducerRecord<>(targetTopic, partition, messageKey, messagePayload); - Future future; - if (producerListener == null) { - future = this.producer.send(record); - } - else { - ProducerListenerInvokingCallback callback = - new ProducerListenerInvokingCallback(targetTopic, partition, messageKey, messagePayload, - producerListener); - future = this.producer.send(record, callback); - } - if (!producerMetadata.isSync()) { - return future; - } - else { - try { - if (producerMetadata.getSendTimeout() <= 0) { - future.get(); - } - else { - future.get(producerMetadata.getSendTimeout(), TimeUnit.MILLISECONDS); - } - } - catch (InterruptedException | ExecutionException | TimeoutException e) { - throw new KafkaException(e); - } - return future; - } - } - - public Future convertAndSend(String topic, Integer partition, Object messageKey, - Object messagePayload) { - return send(topic, partition, convertKeyIfNecessary(messageKey), convertPayloadIfNecessary(messagePayload)); - } - - public Future convertAndSend(String topic, Object messageKey, Object messagePayload) { - return send(topic, convertKeyIfNecessary(messageKey), convertPayloadIfNecessary(messagePayload)); - } - - private K convertKeyIfNecessary(Object messageKey) { - if (messageKey != null) { - if (getProducerMetadata().getKeyClassType().isAssignableFrom( - messageKey.getClass())) { - return getProducerMetadata().getKeyClassType().cast(messageKey); - } - return this.conversionService.convert(messageKey, this.producerMetadata.getKeyClassType()); - } - else { - return null; - } - } - - private V convertPayloadIfNecessary(Object messagePayload) { - if (messagePayload != null) { - if (getProducerMetadata().getValueClassType().isAssignableFrom( - messagePayload.getClass())) { - return getProducerMetadata().getValueClassType().cast(messagePayload); - } - return this.conversionService.convert(messagePayload, this.producerMetadata.getValueClassType()); - } - else { - return null; - } - } - - public void stop() { - this.producer.close(); - } - - @Override - public String toString() { - return "ProducerConfiguration{" + - "producer=" + this.producer + - ", producerMetadata=" + this.producerMetadata + - ", conversionService=" + this.conversionService + - '}'; - } - - private class StringBytesConverter implements GenericConverter { - - @Override - public Set getConvertibleTypes() { - Set convertiblePairs = new HashSet(); - convertiblePairs.add(new ConvertiblePair(String.class, byte[].class)); - convertiblePairs.add(new ConvertiblePair(byte[].class, String.class)); - return convertiblePairs; - } - - @Override - public Object convert(Object source, TypeDescriptor sourceType, TypeDescriptor targetType) { - if (source instanceof String) { - return ((String) source).getBytes(ProducerConfiguration.this.producerMetadata.getCharset()); - } - else { - return new String((byte[]) source, ProducerConfiguration.this.producerMetadata.getCharset()); - } - } - - } - -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ProducerFactoryBean.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ProducerFactoryBean.java deleted file mode 100644 index ece45c5ffc..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ProducerFactoryBean.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.support; - -import java.util.Properties; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.kafka.clients.producer.KafkaProducer; -import org.apache.kafka.clients.producer.Producer; -import org.apache.kafka.clients.producer.ProducerConfig; - -import org.springframework.beans.factory.FactoryBean; - -/** - * @author Soby Chacko - * @author Marius Bogoevici - * @since 0.5 - */ -public class ProducerFactoryBean implements FactoryBean> { - - private static final Log LOGGER = LogFactory.getLog(ProducerFactoryBean.class); - - private final String brokerList; - - private final ProducerMetadata producerMetadata; - - private Properties producerProperties = new Properties(); - - public ProducerFactoryBean(final ProducerMetadata producerMetadata, final String brokerList, - final Properties producerProperties) { - this.producerMetadata = producerMetadata; - this.brokerList = brokerList; - if (producerProperties != null) { - this.producerProperties = producerProperties; - } - } - - public ProducerFactoryBean(final ProducerMetadata producerMetadata, final String brokerList) { - this(producerMetadata, brokerList, null); - } - - @Override - public Producer getObject() throws Exception { - final Properties props = new Properties(); - props.putAll(producerProperties); - props.put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, brokerList); - props.put(ProducerConfig.COMPRESSION_TYPE_CONFIG, producerMetadata.getCompressionType().name()); - props.put(ProducerConfig.BATCH_SIZE_CONFIG, producerMetadata.getBatchBytes()); - LOGGER.info("Using producer properties => " + props); - return new KafkaProducer<>(props, - producerMetadata.getKeySerializer(), - producerMetadata.getValueSerializer()); - } - - @Override - public Class getObjectType() { - return Producer.class; - } - - @Override - public boolean isSingleton() { - return true; - } -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ProducerListener.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ProducerListener.java deleted file mode 100644 index 6b29a9ea3a..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ProducerListener.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.support; - -import org.apache.kafka.clients.producer.RecordMetadata; - -/** - * Listener for handling outbound Kafka messages. Exactly one of its methods will be invoked, depending on whether - * the write has been acknowledged or not. - * - * Its main goal is to provide a stateless singleton delegate for {@link org.apache.kafka.clients.producer.Callback}s, - * which, in all but the most trivial cases, requires creating a separate instance per message. - * - * @see org.apache.kafka.clients.producer.Callback - * @author Marius Bogoevici - * @since 1.3 - */ -public interface ProducerListener { - - /** - * Invoked after the successful send of a message (that is, after it has been acknowledged by the broker) - * - * @param topic the destination topic - * @param partition the destination partition (could be null) - * @param key the key of the outbound message - * @param value the payload of the outbound message - * @param recordMetadata the result of the successful send operation - */ - void onSuccess(String topic, Integer partition, Object key, Object value, RecordMetadata recordMetadata); - - /** - * Invoked after an attempt to send a message has failed - * - * @param topic the destination topic - * @param partition the destination partition (could be null) - * @param key the key of the outbound message - * @param value the payload of the outbound message - * @param exception the exception thrown - */ - void onError(String topic, Integer partition, Object key, Object value,Exception exception); -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ProducerListenerInvokingCallback.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ProducerListenerInvokingCallback.java deleted file mode 100644 index 59cb267f4d..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ProducerListenerInvokingCallback.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.support; - -import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.clients.producer.RecordMetadata; - -import org.springframework.util.Assert; - -/** - * Adapts the {@link org.apache.kafka.clients.producer.Callback} interface of the - * {@link org.apache.kafka.clients.producer.Producer} to a {@link ProducerListener}. - * - * @author Marius Bogoevici - */ -public class ProducerListenerInvokingCallback implements Callback { - - private final String topic; - - private final Integer partition; - - private final Object key; - - private final Object payload; - - private final ProducerListener producerListener; - - public ProducerListenerInvokingCallback(String topic, Integer partition, Object key, Object payload, - ProducerListener producerListener) { - Assert.notNull(producerListener, "must not be null"); - this.topic = topic; - this.partition = partition; - this.key = key; - this.payload = payload; - this.producerListener = producerListener; - } - - @Override - public void onCompletion(RecordMetadata metadata, Exception exception) { - if (exception != null) { - producerListener.onError(topic,partition,key, payload,exception); - } - else { - producerListener.onSuccess(topic, partition, key, payload, metadata); - } - } -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ProducerMetadata.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ProducerMetadata.java deleted file mode 100644 index 2494c983dd..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ProducerMetadata.java +++ /dev/null @@ -1,173 +0,0 @@ -/* - * Copyright 2002-2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.support; - -import java.nio.charset.Charset; - -import org.apache.kafka.common.serialization.Serializer; - -import org.springframework.util.Assert; - -import kafka.producer.Partitioner; - -/** - * - * @author Soby Chacko - * @author Rajasekar Elango - * @author Marius Bogoevici - * @author Artem Bilan - * - * @since 0.5 - */ -public class ProducerMetadata { - - private final Class keyClassType; - - private final Class valueClassType; - - private Serializer keySerializer; - - private Serializer valueSerializer; - - private final String topic; - - private Partitioner partitioner; - - private CompressionType compressionType = CompressionType.none; - - private int batchBytes = 16384; - - private int sendTimeout = 0; - - private boolean sync = false; - - private Charset charset = Charset.forName("UTF8"); - - public ProducerMetadata(final String topic, Class keyClassType, Class valueClassType, - Serializer keySerializer, Serializer valueSerializer) { - Assert.notNull(topic, "Topic cannot be null"); - Assert.notNull(keyClassType, "Key class type serializer cannot be null"); - Assert.notNull(valueClassType, "Value class type cannot be null"); - Assert.notNull(keySerializer, "Value serializer cannot be null"); - Assert.notNull(valueSerializer, "Value serializer cannot be null"); - this.topic = topic; - this.keyClassType = keyClassType; - this.valueClassType = valueClassType; - this.keySerializer = keySerializer; - this.valueSerializer = valueSerializer; - } - - public String getTopic() { - return topic; - } - - public Serializer getKeySerializer() { - return keySerializer; - } - - public Serializer getValueSerializer() { - return valueSerializer; - } - - public CompressionType getCompressionType() { - return compressionType; - } - - public void setCompressionType(CompressionType compressionType) { - Assert.notNull(compressionType, "Compression type cannot be null"); - this.compressionType = compressionType; - } - - public int getBatchBytes() { - return batchBytes; - } - - public void setBatchBytes(int batchBytes) { - Assert.isTrue(batchBytes > 0, "Buffer size must be greater than zero"); - this.batchBytes = batchBytes; - } - - public Partitioner getPartitioner() { - return partitioner; - } - - public void setPartitioner(Partitioner partitioner) { - this.partitioner = partitioner; - } - - public Class getKeyClassType() { - return keyClassType; - } - - public Class getValueClassType() { - return valueClassType; - } - - public int getSendTimeout() { - return sendTimeout; - } - - public void setSendTimeout(int sendTimeout) { - this.sendTimeout = sendTimeout; - } - - public boolean isSync() { - return sync; - } - - public void setSync(boolean sync) { - this.sync = sync; - } - - public Charset getCharset() { - return charset; - } - - /** - * The character encoding to preform {@code String <-> byte[]} conversion - * instead of general (de)serialization. - * @param charset the charset encoding to use. - * @since 1.3 - */ - public void setCharset(Charset charset) { - this.charset = charset; - } - - @Override - public String toString() { - return "ProducerMetadata{" + - "keyClassType=" + this.keyClassType + - ", valueClassType=" + this.valueClassType + - ", keySerializer=" + this.keySerializer + - ", valueSerializer=" + this.valueSerializer + - ", topic='" + this.topic + '\'' + - ", partitioner=" + this.partitioner + - ", compressionType=" + this.compressionType + - ", batchBytes=" + this.batchBytes + - ", sync=" + this.sync + - ", sendTimeout=" + this.sendTimeout + - ", charset=" + this.charset + - '}'; - } - - public enum CompressionType { - none, - gzip, - snappy - } - -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/TopicFilterConfiguration.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/TopicFilterConfiguration.java deleted file mode 100644 index 16052542e2..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/TopicFilterConfiguration.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright 2002-2014 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.support; - -import kafka.consumer.Blacklist; -import kafka.consumer.TopicFilter; -import kafka.consumer.Whitelist; - -/** - * @author Rajasekar Elango - * @author Artem Bilan - * @since 0.5 - */ -@Deprecated -public class TopicFilterConfiguration { - - private final int numberOfStreams; - - private final TopicFilter topicFilter; - - public TopicFilterConfiguration(final String pattern, final int numberOfStreams, final boolean exclude) { - this.numberOfStreams = numberOfStreams; - if (exclude) { - this.topicFilter = new Blacklist(pattern); - } - else { - this.topicFilter = new Whitelist(pattern); - } - } - - public TopicFilter getTopicFilter() { - return this.topicFilter; - } - - public int getNumberOfStreams() { - return this.numberOfStreams; - } - - @Override - public String toString() { - return this.topicFilter.toString() + " : " + this.numberOfStreams; - } - -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ZookeeperConnect.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ZookeeperConnect.java deleted file mode 100644 index cb52e28600..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/ZookeeperConnect.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.kafka.support; - -import org.springframework.integration.kafka.core.ZookeeperConnectDefaults; - -/** - * @author Soby Chacko - * @since 0.5 - */ -public class ZookeeperConnect { - - private String zkConnect = ZookeeperConnectDefaults.ZK_CONNECT; - - private String zkConnectionTimeout = ZookeeperConnectDefaults.ZK_CONNECTION_TIMEOUT; - - private String zkSessionTimeout = ZookeeperConnectDefaults.ZK_SESSION_TIMEOUT; - - private String zkSyncTime = ZookeeperConnectDefaults.ZK_SYNC_TIME; - - public ZookeeperConnect() { - } - - public ZookeeperConnect(String zkConnect) { - this.zkConnect = zkConnect; - } - - public String getZkConnect() { - return zkConnect; - } - - public void setZkConnect(final String zkConnect) { - this.zkConnect = zkConnect; - } - - public String getZkConnectionTimeout() { - return zkConnectionTimeout; - } - - public void setZkConnectionTimeout(final String zkConnectionTimeout) { - this.zkConnectionTimeout = zkConnectionTimeout; - } - - public String getZkSessionTimeout() { - return zkSessionTimeout; - } - - public void setZkSessionTimeout(final String zkSessionTimeout) { - this.zkSessionTimeout = zkSessionTimeout; - } - - public String getZkSyncTime() { - return zkSyncTime; - } - - public void setZkSyncTime(final String zkSyncTime) { - this.zkSyncTime = zkSyncTime; - } -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/package-info.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/package-info.java deleted file mode 100644 index 0ec591dea7..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/support/package-info.java +++ /dev/null @@ -1,4 +0,0 @@ -/** - * Provides various support classes used across Spring Integration Kafka Components. - */ -package org.springframework.integration.kafka.support; diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/util/EncoderAdaptingSerializer.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/util/EncoderAdaptingSerializer.java deleted file mode 100644 index e32c52cbff..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/util/EncoderAdaptingSerializer.java +++ /dev/null @@ -1,57 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.util; - -import java.util.Map; - -import kafka.serializer.DefaultEncoder; -import kafka.serializer.Encoder; -import org.apache.kafka.common.serialization.Serializer; - -/** - * An adapter from the pre-0.8.2 Kafka {@link Encoder} to the {@link Serializer} interface used - * by the new client. - * - * @author Marius Bogoevici - */ -public class EncoderAdaptingSerializer implements Serializer { - - private final Encoder encoder; - - public EncoderAdaptingSerializer(Encoder encoder) { - this.encoder = encoder; - } - - public Encoder getEncoder() { - return encoder; - } - - @Override - public void configure(Map configs, boolean isKey) { - // no-op - } - - @Override - public byte[] serialize(String topic, T data) { - return encoder.toBytes(data); - } - - @Override - public void close() { - // no-op; - } -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/util/LoggingUtils.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/util/LoggingUtils.java deleted file mode 100644 index a5d775e1bf..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/util/LoggingUtils.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.util; - -/** - * Utilities for logging data - * - * @author Marius Bogoevici - */ -public class LoggingUtils { - - public static String asCommaSeparatedHexDump(byte[] bytes) { - if (bytes == null || bytes.length == 0) { - return "[]"; - } - else if (bytes.length == 1) { - return String.format("[%s]", Integer.toHexString(bytes[0])); - } - else { - StringBuilder buffer = new StringBuilder("["); - for (int i = 0; i < bytes.length - 1; i++) { - buffer.append(Integer.toHexString(bytes[i])); - buffer.append(","); - } - buffer.append(Integer.toHexString(bytes[bytes.length])); - buffer.append("]"); - return buffer.toString(); - } - } - -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/util/MessageUtils.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/util/MessageUtils.java deleted file mode 100644 index 94c702b0dc..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/util/MessageUtils.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.util; - -import org.springframework.integration.kafka.core.KafkaMessage; - -import kafka.serializer.Decoder; -import kafka.utils.Utils$; - -/** - * @author Marius Bogoevici - */ -public class MessageUtils { - - public static T decodeKey(KafkaMessage message, Decoder decoder) { - if (!message.getMessage().hasKey()) { - return null; - } - return decoder.fromBytes(Utils$.MODULE$.readBytes(message.getMessage().key())); - } - - public static T decodePayload(KafkaMessage message, Decoder decoder) { - if (message.getMessage().isNull()) { - return null; - } - return decoder.fromBytes(Utils$.MODULE$.readBytes(message.getMessage().payload())); - } - -} diff --git a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/util/TopicUtils.java b/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/util/TopicUtils.java deleted file mode 100644 index 19c3dfe8bd..0000000000 --- a/spring-integration-kafka/src/main/java/org/springframework/integration/kafka/util/TopicUtils.java +++ /dev/null @@ -1,160 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.util; - -import java.util.List; -import java.util.Properties; - -import kafka.common.LeaderNotAvailableException; -import org.I0Itec.zkclient.ZkClient; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import org.springframework.integration.kafka.core.TopicNotFoundException; -import org.springframework.retry.RetryCallback; -import org.springframework.retry.RetryContext; -import org.springframework.retry.RetryPolicy; -import org.springframework.retry.backoff.ExponentialBackOffPolicy; -import org.springframework.retry.policy.CompositeRetryPolicy; -import org.springframework.retry.policy.SimpleRetryPolicy; -import org.springframework.retry.policy.TimeoutRetryPolicy; -import org.springframework.retry.support.RetryTemplate; - -import kafka.admin.AdminUtils; -import kafka.api.TopicMetadata; -import kafka.common.ErrorMapping; -import kafka.javaapi.PartitionMetadata; -import kafka.utils.ZKStringSerializer$; -import kafka.utils.ZkUtils; -import scala.collection.Map; -import scala.collection.Seq; - -/** - * Utilities for interacting with Kafka topics - * - * @author Marius Bogoevici - */ -public class TopicUtils { - - private static Log log = LogFactory.getLog(TopicUtils.class); - - public static final int METADATA_VERIFICATION_TIMEOUT = 5000; - - public static final int METADATA_VERIFICATION_RETRY_ATTEMPTS = 10; - - public static final double METADATA_VERIFICATION_RETRY_BACKOFF_MULTIPLIER = 1.5; - - public static final int METADATA_VERIFICATION_RETRY_INITIAL_INTERVAL = 100; - - public static final int METADATA_VERIFICATION_MAX_INTERVAL = 1000; - - /** - * Creates a topic in Kafka or validates that it exists with the requested number of partitions, - * and returns only after the topic has been fully created - * @param zkAddress the address of the Kafka ZooKeeper instance - * @param topicName the name of the topic - * @param numPartitions the number of partitions - * @param replicationFactor the replication factor - * @return {@link TopicMetadata} information for the topic - */ - public static TopicMetadata ensureTopicCreated(final String zkAddress, final String topicName, - final int numPartitions, int replicationFactor) { - - final int sessionTimeoutMs = 10000; - final int connectionTimeoutMs = 10000; - final ZkClient zkClient = new ZkClient(zkAddress, sessionTimeoutMs, connectionTimeoutMs, - ZKStringSerializer$.MODULE$); - try { - // The following is basically copy/paste from AdminUtils.createTopic() with - // createOrUpdateTopicPartitionAssignmentPathInZK(..., update=true) - Properties topicConfig = new Properties(); - Seq brokerList = ZkUtils.getSortedBrokerList(zkClient); - scala.collection.Map> replicaAssignment = - AdminUtils.assignReplicasToBrokers(brokerList, numPartitions, replicationFactor, -1, -1); - return ensureTopicCreated(zkClient, topicName, numPartitions, topicConfig, replicaAssignment); - - } - finally { - zkClient.close(); - } - } - - /** - * Creates a topic in Kafka and returns only after the topic has been fully and an produce metadata. - * @param zkClient an open {@link ZkClient} connection to Zookeeper - * @param topicName the name of the topic - * @param numPartitions the number of partitions for the topic - * @param topicConfig additional topic configuration properties - * @param replicaAssignment the mapping of partitions to broker - * @return {@link TopicMetadata} information for the topic - */ - public static TopicMetadata ensureTopicCreated(final ZkClient zkClient, final String topicName, - final int numPartitions, Properties topicConfig, Map> replicaAssignment) { - AdminUtils.createOrUpdateTopicPartitionAssignmentPathInZK(zkClient, topicName, replicaAssignment, topicConfig, - true); - - RetryTemplate retryTemplate = new RetryTemplate(); - - CompositeRetryPolicy policy = new CompositeRetryPolicy(); - TimeoutRetryPolicy timeoutRetryPolicy = new TimeoutRetryPolicy(); - timeoutRetryPolicy.setTimeout(METADATA_VERIFICATION_TIMEOUT); - SimpleRetryPolicy simpleRetryPolicy = new SimpleRetryPolicy(); - simpleRetryPolicy.setMaxAttempts(METADATA_VERIFICATION_RETRY_ATTEMPTS); - policy.setPolicies(new RetryPolicy[] {timeoutRetryPolicy, simpleRetryPolicy}); - retryTemplate.setRetryPolicy(policy); - - ExponentialBackOffPolicy backOffPolicy = new ExponentialBackOffPolicy(); - backOffPolicy.setInitialInterval(METADATA_VERIFICATION_RETRY_INITIAL_INTERVAL); - backOffPolicy.setMultiplier(METADATA_VERIFICATION_RETRY_BACKOFF_MULTIPLIER); - backOffPolicy.setMaxInterval(METADATA_VERIFICATION_MAX_INTERVAL); - retryTemplate.setBackOffPolicy(backOffPolicy); - - try { - return retryTemplate.execute(new RetryCallback() { - @Override - public TopicMetadata doWithRetry(RetryContext context) throws Exception { - TopicMetadata topicMetadata = AdminUtils.fetchTopicMetadataFromZk(topicName, zkClient); - if (topicMetadata.errorCode() != ErrorMapping.NoError() || - !topicName.equals(topicMetadata.topic())) { - // downcast to Exception because that's what the error throws - throw (Exception) ErrorMapping.exceptionFor(topicMetadata.errorCode()); - } - List partitionMetadatas = - new kafka.javaapi.TopicMetadata(topicMetadata).partitionsMetadata(); - if (partitionMetadatas.size() != numPartitions) { - throw new IllegalStateException("The number of expected partitions was: " + - numPartitions + ", but " + partitionMetadatas.size() + " have been found instead"); - } - for (PartitionMetadata partitionMetadata : partitionMetadatas) { - if (partitionMetadata.errorCode() != ErrorMapping.NoError()) { - throw (Exception) ErrorMapping.exceptionFor(partitionMetadata.errorCode()); - } - if (partitionMetadata.leader() == null) { - throw new LeaderNotAvailableException(); - } - } - return topicMetadata; - } - }); - } - catch (Exception e) { - log.error(String.format("Cannot retrieve metadata for topic '%s'", topicName), e); - throw new TopicNotFoundException(topicName); - } - } - -} diff --git a/spring-integration-kafka/src/main/resources/org/springframework/integration/kafka/config/spring-integration-kafka-2.0.xsd b/spring-integration-kafka/src/main/resources/org/springframework/integration/kafka/config/spring-integration-kafka-2.0.xsd index 30fd9f614a..96208456b8 100644 --- a/spring-integration-kafka/src/main/resources/org/springframework/integration/kafka/config/spring-integration-kafka-2.0.xsd +++ b/spring-integration-kafka/src/main/resources/org/springframework/integration/kafka/config/spring-integration-kafka-2.0.xsd @@ -14,479 +14,15 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - The topic configured by this configuration. - - - - - - - - - - - - - Class type used for the key - - - - - - - - - - - - Class type used for the value - - - - - - - - - - - - [DEPRECATED] - Custom implementation of a Kafka Encoder for encoding message values. - This option is deprecated, 'value-serializer' is the recommended option. - - - - - - - - - - - - Custom implementation of a Kafka Serializer for message values. - - - - - - - - - - - - [DEPRECATED] - Custom implementation of a Kafka Encoder for encoding message keys. - This option is deprecated, 'key-serializer' is the recommended option. - - - - - - - - - - - - Custom implementation of a Kafka Serializer for message keys. - - - - - - - - - - - - Conversion service for transforming inbound objects into the target key and value - - - - - - - - - - - - Listener for notifying when a message has been sent or has failed. - - - - - - - - - - - - - - - - - [DEPRECATED] - Custom Kafka key partitioner. - Deprecated in favor of 'partition-id' ('partition-id-expression') - on the 'outbound-channel-adapter'. - - - - - - - - - - - number of messages to batch at this producer. - - - - - - - If true, sends messages synchronously. Asynchronously (default) otherwise - - - - - - - If sync=true, specify timeout in milliseconds. 0 or below indicate forever (default) - - - - - - - The character encoding to preform String to/from byte[] conversion - instead of general (de)serialization. Defaults to UTF8. - - - - - - - - - - - - - - - Kafka producer properties to use for all producers - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Regex pattern to match topic - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Custom implementation of a Kafka Decoder for values. - - - - - - - - - - - - Custom implementation of a Kafka Decoder for keys. - - - - - - - - - - - - A java.util.concurrent.Executor bean reference. - Typically - org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor. - Used to execute iterator task on KafkaStreams. - - - - - - - - - - - - - - - - - - - - - - - - - Kafka Server Bean Name - - - - - - - - - - - - Kafka consumer properties to use for all consumers - - - - - - - - - - - - - - [DEPRECATED] - The definition for the Spring Integration Kafka - Inbound Channel Adapter. - This High Level Consumer Adapter is deprecated in favor of - message-driven-channel-adapter - based on the Simple Consumer API. - - - - - - - - - - - Maximum amount of time in milliseconds to wait when sending a message to the channel - if such channel may block. For example, a Queue Channel can block until space is available - if its maximum capacity has been reached. - - - - - - - - - - - - - Kafka consumer context reference. - - - - - - - - - - - Defines kafka outbound channel adapter that writes the contents of the - Message to kafka broker. + Defines the Consumer Endpoint for the KafkaProducerMessageHandler + that writes the contents of the Message to kafka broker. @@ -497,14 +33,14 @@ minOccurs="0" maxOccurs="1" /> - + + - - Kafka producer context reference. - - + @@ -585,8 +121,7 @@ - The definition for the Spring Integration Kafka - KafkaMessageDrivenChannelAdapter. + Defines the Message Producing Endpoint for the KafkaMessageDrivenChannelAdapter. @@ -612,170 +147,18 @@ - - - - A 'kafka.serializer.Decoder' bean reference for decoding the 'key' - of the received the 'kafka.message.Message'. Defaults to 'kafka.serializer.DefaultDecoder' - without 'kafka.utils.VerifiableProperties'. - - - - - - - - - - - - A 'kafka.serializer.Decoder' bean reference for decoding the 'payload' - of the received the 'kafka.message.Message'. Defaults to 'kafka.serializer.DefaultDecoder' - without 'kafka.utils.VerifiableProperties'. - - - - - - - - - A 'org.springframework.integration.kafka.listener.KafkaMessageListenerContainer' bean reference. - Mutually exclusive with 'connection-factory', 'topics', 'offset-manager', 'error-handler', - 'task-executor', 'concurrency', 'stop-timeout', 'max-fetch', 'queue-size'. + An 'org.springframework.kafka.listener.AbstractMessageListenerContainer' bean reference. - + - - - - A 'org.springframework.integration.kafka.core.ConnectionFactory' bean reference. - Mutually exclusive with 'listener-container'. - - - - - - - - - - - - The comma-separated Kafka topics to listen to. - Mutually exclusive with 'listener-container'. - - - - - - - A 'org.springframework.integration.kafka.listener.OffsetManager' bean reference. - Defaults to 'org.springframework.integration.kafka.listener.MetadataStoreOffsetManager'. - Mutually exclusive with 'listener-container'. - - - - - - - - - - - - A 'org.springframework.integration.kafka.listener.ErrorHandler' bean reference. - Used from 'org.springframework.integration.kafka.listener.KafkaMessageListenerContainer' - to handle errors from 'kafka.message.Message' decoding and when the 'error-channel' - isn't provided to catch message producing errors. - Defaults to 'org.springframework.integration.kafka.listener.LoggingErrorHandler. - Mutually exclusive with 'listener-container'. - - - - - - - - - - - - A 'java.util.concurrent.Executor' bean reference to fetch messages from Kafka. - Defaults to the 'ThreadPoolExecutor' with the pool based on the number of 'partitionsByBroker'. - Mutually exclusive with 'listener-container'. - - - - - - - - - - - - The number of internal concurrent - 'org.springframework.integration.kafka.listener.QueueingMessageListenerInvoker'. - Defaults to '1'. - Mutually exclusive with 'listener-container'. - - - - - - - The maximum amount of time (in milliseconds) to wait for each 'org.springframework.integration.kafka.listener.QueueingMessageListenerInvoker' to finish before stopping. - Defaults to '1000'. - Mutually exclusive with 'listener-container'. - - - - - - - The maximum amount of data (in bytes) that pollers will fetch in one round. - Defaults to '300 * 1024'. - Mutually exclusive with 'listener-container'. - - - - - - - The maximum number of messages that are buffered by each concurrent 'MessageListener' runner. - Increasing the value may increase throughput, but also increases the memory consumption. - Defaults to '1024'. - Mutually exclusive with 'listener-container'. - - - - - - - When 'true', the adapter automatically commits the offset (also see `auto-commit-on-error`). - When 'false', the adapter inserts a 'kafka_acknowledgment` header allowing the user to manually - commit the offset using the `Acknowledgment.acknowledge()` method. Default 'true'. - - - - - - - When 'true', the adapter automatically auto-commits only the offsets of successfully processed - messages if `auto-commit` is set to true. This means that only the offset of the last - successfully processed message is committed. - - - diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaConsumerContextParserTests-context.xml b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaConsumerContextParserTests-context.xml deleted file mode 100644 index d787a2dc5a..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaConsumerContextParserTests-context.xml +++ /dev/null @@ -1,54 +0,0 @@ - - - - - - - - - - - - - - largest - 10485760 - 5242880 - 1000 - - - - - - foo - 10 - true - - - - - - - - - - - - - diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaConsumerContextParserTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaConsumerContextParserTests.java deleted file mode 100644 index dd0214a418..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaConsumerContextParserTests.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright 2013-2014 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.config.xml; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThat; - -import org.hamcrest.Matchers; -import org.junit.Assert; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.runner.RunWith; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; -import org.springframework.integration.kafka.rule.KafkaRunning; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; - -import kafka.consumer.Blacklist; - -/** - * @author Soby Chacko - * @author Artem Bilan - * @author Gary Russell - * @since 0.5 - */ -@RunWith(SpringJUnit4ClassRunner.class) -@ContextConfiguration -@Deprecated -@SuppressWarnings("deprecation") -public class KafkaConsumerContextParserTests { - - @ClassRule - public static KafkaRunning kafkaRunning = KafkaRunning.isRunning(); - - @Autowired - private ApplicationContext appContext; - - @Test - @SuppressWarnings("unchecked") - public void testConsumerContextConfiguration() { - final org.springframework.integration.kafka.support.KafkaConsumerContext consumerContext = - appContext.getBean("consumerContext", - org.springframework.integration.kafka.support.KafkaConsumerContext.class); - Assert.assertNotNull(consumerContext); - org.springframework.integration.kafka.support.ConsumerConfiguration cc - = consumerContext.getConsumerConfiguration("default1"); - org.springframework.integration.kafka.support.ConsumerMetadata cm = cc.getConsumerMetadata(); - assertNotNull(cm); - org.springframework.integration.kafka.support.TopicFilterConfiguration topicFilterConfiguration = - cm.getTopicFilterConfiguration(); - assertEquals("foo : 10", topicFilterConfiguration.toString()); - assertThat(topicFilterConfiguration.getTopicFilter(), Matchers.instanceOf(Blacklist.class)); - } - -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaInboundAdapterParserTests-context.xml b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaInboundAdapterParserTests-context.xml deleted file mode 100644 index b1ac3cfa4e..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaInboundAdapterParserTests-context.xml +++ /dev/null @@ -1,40 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaInboundAdapterParserTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaInboundAdapterParserTests.java deleted file mode 100644 index f15908d686..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaInboundAdapterParserTests.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright 2013-2014 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.kafka.config.xml; - -import org.junit.Assert; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.runner.RunWith; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; -import org.springframework.integration.endpoint.SourcePollingChannelAdapter; -import org.springframework.integration.kafka.rule.KafkaRunning; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; - -/** - * @author Soby Chacko - * @author Gary Russell - * @since 0.5 - */ -@RunWith(SpringJUnit4ClassRunner.class) -@ContextConfiguration -@Deprecated -@SuppressWarnings("deprecation") -public class KafkaInboundAdapterParserTests { - - @ClassRule - public static KafkaRunning kafkaRunning = KafkaRunning.isRunning(); - - @Autowired - private ApplicationContext appContext; - - /** - * Test method for {@link org.springframework.integration.kafka.config.xml.KafkaInboundChannelAdapterParser#parseSource(org.w3c.dom.Element, org.springframework.beans.factory.xml.ParserContext)}. - */ - @Test - public void testParseSourceElementParserContext() throws Exception { - final SourcePollingChannelAdapter adapter = appContext.getBean("kafkaInboundChannelAdapter", - SourcePollingChannelAdapter.class); - - Assert.assertNotNull(adapter); - Assert.assertFalse(adapter.isAutoStartup()); - } -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaMessageDrivenChannelAdapterParserTests-context.xml b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaMessageDrivenChannelAdapterParserTests-context.xml index 6dd12501fb..7f68dc5894 100644 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaMessageDrivenChannelAdapterParserTests-context.xml +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaMessageDrivenChannelAdapterParserTests-context.xml @@ -1,134 +1,38 @@ + http://www.springframework.org/schema/integration/kafka http://www.springframework.org/schema/integration/kafka/spring-integration-kafka.xsd + http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd"> - - - - - - - - - - - - - - - - - - - - - - + error-channel="errorChannel" /> - - - - - - - - - diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaMessageDrivenChannelAdapterParserTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaMessageDrivenChannelAdapterParserTests.java index 016b89afe4..958f54d9e4 100644 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaMessageDrivenChannelAdapterParserTests.java +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaMessageDrivenChannelAdapterParserTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2015 the original author or authors + * Copyright 2015-2016 the original author or authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,47 +16,22 @@ package org.springframework.integration.kafka.config.xml; -import static org.hamcrest.Matchers.equalTo; -import static org.junit.Assert.assertArrayEquals; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertNull; import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertThat; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; -import java.lang.reflect.Method; -import java.util.concurrent.Executor; -import java.util.concurrent.atomic.AtomicReference; - -import org.hamcrest.Matchers; import org.junit.Test; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.integration.channel.NullChannel; import org.springframework.integration.channel.PublishSubscribeChannel; -import org.springframework.integration.kafka.core.ConnectionFactory; -import org.springframework.integration.kafka.core.KafkaMessageMetadata; -import org.springframework.integration.kafka.core.Partition; import org.springframework.integration.kafka.inbound.KafkaMessageDrivenChannelAdapter; -import org.springframework.integration.kafka.listener.Acknowledgment; -import org.springframework.integration.kafka.listener.ErrorHandler; -import org.springframework.integration.kafka.listener.KafkaMessageListenerContainer; -import org.springframework.integration.kafka.listener.OffsetManager; -import org.springframework.integration.kafka.support.KafkaHeaders; import org.springframework.integration.test.util.TestUtils; -import org.springframework.messaging.Message; -import org.springframework.messaging.support.GenericMessage; +import org.springframework.kafka.listener.KafkaMessageListenerContainer; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; -import org.springframework.util.ReflectionUtils; -import org.springframework.util.ReflectionUtils.MethodCallback; -import org.springframework.util.ReflectionUtils.MethodFilter; - -import kafka.serializer.Decoder; /** * @author Artem Bilan. @@ -66,24 +41,6 @@ import kafka.serializer.Decoder; @ContextConfiguration public class KafkaMessageDrivenChannelAdapterParserTests { - @Autowired - private ConnectionFactory connectionFactory; - - @Autowired - private OffsetManager offsetManager; - - @Autowired - private ErrorHandler errorHandler; - - @Autowired - private Executor executor; - - @Autowired - private Decoder keyDecoder; - - @Autowired - private Decoder payloadDecoder; - @Autowired private NullChannel nullChannel; @@ -91,16 +48,7 @@ public class KafkaMessageDrivenChannelAdapterParserTests { private PublishSubscribeChannel errorChannel; @Autowired - private KafkaMessageDrivenChannelAdapter kafkaListener; - - @Autowired - private KafkaMessageDrivenChannelAdapter withMBFactoryOverrideAndId; - - @Autowired - private KafkaMessageDrivenChannelAdapter withMBFactoryOverrideAndTS; - - @Autowired - private KafkaMessageDrivenChannelAdapter withOverrideIdTS; + private KafkaMessageDrivenChannelAdapter kafkaListener; @Test public void testKafkaMessageDrivenChannelAdapterParser() throws Exception { @@ -109,99 +57,10 @@ public class KafkaMessageDrivenChannelAdapterParserTests { assertEquals(100, this.kafkaListener.getPhase()); assertSame(this.nullChannel, TestUtils.getPropertyValue(this.kafkaListener, "outputChannel")); assertSame(this.errorChannel, TestUtils.getPropertyValue(this.kafkaListener, "errorChannel")); - assertSame(this.keyDecoder, TestUtils.getPropertyValue(this.kafkaListener, "keyDecoder")); - assertSame(this.payloadDecoder, TestUtils.getPropertyValue(this.kafkaListener, "payloadDecoder")); - KafkaMessageListenerContainer container = + KafkaMessageListenerContainer container = TestUtils.getPropertyValue(this.kafkaListener, "messageListenerContainer", KafkaMessageListenerContainer.class); - assertSame(this.connectionFactory, TestUtils.getPropertyValue(container, "kafkaTemplate.connectionFactory")); - assertSame(this.offsetManager, container.getOffsetManager()); - assertSame(this.errorHandler, container.getErrorHandler()); - assertSame(this.executor, container.getFetchTaskExecutor()); - assertEquals(10, container.getConcurrency()); - assertEquals(1000, container.getMaxFetch()); - assertEquals(1024, container.getQueueSize()); - assertEquals(5000, container.getStopTimeout()); - assertArrayEquals(new String[] {"foo", "bar"}, TestUtils.getPropertyValue(container, "topics", String[].class)); - assertOverrides(this.kafkaListener, false, false, false, true, false); - assertOverrides(this.withMBFactoryOverrideAndId, true, true, false, false, false); - assertOverrides(this.withMBFactoryOverrideAndTS, true, false, true, true, false); - assertOverrides(this.withOverrideIdTS, false, true, true, true, true); - - - final AtomicReference toMessage = new AtomicReference(); - ReflectionUtils.doWithMethods(KafkaMessageDrivenChannelAdapter.class, new MethodCallback() { - - @Override - public void doWith(Method method) throws IllegalArgumentException, IllegalAccessException { - method.setAccessible(true); - toMessage.set(method); - } - }, - new MethodFilter() { - - @Override - public boolean matches(Method method) { - return method.getName().equals("toMessage"); - } - }); - - Message m = getAMessageFrom(this.kafkaListener, toMessage.get()); - assertNull(m.getHeaders().getId()); - assertNull(m.getHeaders().getTimestamp()); - assertNull(m.getHeaders().get(KafkaHeaders.ACKNOWLEDGMENT)); - assertRest(m); - - m = getAMessageFrom(this.withMBFactoryOverrideAndId, toMessage.get()); - assertThat(m, Matchers.instanceOf(GenericMessage.class)); - assertNotNull(m.getHeaders().getId()); - assertNotNull(m.getHeaders().getTimestamp()); - assertNotNull(m.getHeaders().get(KafkaHeaders.ACKNOWLEDGMENT)); - assertRest(m); - - m = getAMessageFrom(this.withMBFactoryOverrideAndTS, toMessage.get()); - assertThat(m, Matchers.instanceOf(GenericMessage.class)); - assertNotNull(m.getHeaders().getId()); - assertNotNull(m.getHeaders().getTimestamp()); - assertNull(m.getHeaders().get(KafkaHeaders.ACKNOWLEDGMENT)); - assertRest(m); - - m = getAMessageFrom(this.withOverrideIdTS, toMessage.get()); - assertNotNull(m.getHeaders().getId()); - assertNotNull(m.getHeaders().getTimestamp()); - assertNull(m.getHeaders().get(KafkaHeaders.ACKNOWLEDGMENT)); - assertRest(m); - } - - private void assertOverrides(KafkaMessageDrivenChannelAdapter kafkaListener, boolean mbf, boolean id, boolean ts, - boolean ac, boolean ace) { - assertThat(TestUtils.getPropertyValue(kafkaListener, "autoCommitOffset", Boolean.class), equalTo(ac)); - assertThat(TestUtils.getPropertyValue(kafkaListener, "useMessageBuilderFactory", Boolean.class), equalTo(mbf)); - assertThat(TestUtils.getPropertyValue(kafkaListener, "generateMessageId", Boolean.class), equalTo(id)); - assertThat(TestUtils.getPropertyValue(kafkaListener, "generateTimestamp", Boolean.class), equalTo(ts)); - Object messageListenerContainer = TestUtils.getPropertyValue(kafkaListener, "messageListenerContainer"); - assertEquals(ace, TestUtils.getPropertyValue(messageListenerContainer,"autoCommitOnError", Boolean.class)); - } - - private void assertRest(Message m) { - assertEquals("bar", m.getPayload()); - assertEquals("foo", m.getHeaders().get(KafkaHeaders.MESSAGE_KEY)); - assertEquals("topic", m.getHeaders().get(KafkaHeaders.TOPIC)); - assertEquals(42, m.getHeaders().get(KafkaHeaders.PARTITION_ID)); - assertEquals(1L, m.getHeaders().get(KafkaHeaders.OFFSET)); - assertEquals(2L, m.getHeaders().get(KafkaHeaders.NEXT_OFFSET)); - } - - private Message getAMessageFrom(KafkaMessageDrivenChannelAdapter adapter, Method toMessage) throws Exception { - KafkaMessageMetadata meta = mock(KafkaMessageMetadata.class); - Partition partition = mock(Partition.class); - when(partition.getTopic()).thenReturn("topic"); - when(partition.getId()).thenReturn(42); - when(meta.getPartition()).thenReturn(partition); - when(meta.getOffset()).thenReturn(1L); - when(meta.getNextOffset()).thenReturn(2L); - Acknowledgment ack = mock(Acknowledgment.class); - return (Message) toMessage.invoke(adapter, "foo", "bar", meta, ack); + assertNotNull(container); } } diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaMultiConsumerContextParserTests-context.xml b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaMultiConsumerContextParserTests-context.xml deleted file mode 100644 index a428f6f284..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaMultiConsumerContextParserTests-context.xml +++ /dev/null @@ -1,69 +0,0 @@ - - - - - - - - - - - - - - - largest - 10485760 - - 5242880 - 1000 - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaMultiConsumerContextParserTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaMultiConsumerContextParserTests.java deleted file mode 100644 index 1ad682ff6b..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaMultiConsumerContextParserTests.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright 2014 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.kafka.config.xml; - -import org.junit.Assert; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; -import org.springframework.integration.kafka.rule.KafkaRunning; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; - -/** - * @author Ilayaperumal Gopinathan - * @author Gary Russell - */ -@RunWith(SpringJUnit4ClassRunner.class) -@ContextConfiguration -@Deprecated -@SuppressWarnings("deprecation") -public class KafkaMultiConsumerContextParserTests { - - @ClassRule - public static KafkaRunning kafkaRunning = KafkaRunning.isRunning(); - - @Autowired - private ApplicationContext appContext; - - @SuppressWarnings("unchecked") - @Test - public void testMultiConsumerContexts() { - final org.springframework.integration.kafka.support.KafkaConsumerContext consumerContext1 = - appContext.getBean("consumerContext1", - org.springframework.integration.kafka.support.KafkaConsumerContext.class); - Assert.assertNotNull(consumerContext1); - final org.springframework.integration.kafka.support.KafkaConsumerContext consumerContext2 = - appContext.getBean("consumerContext2", - org.springframework.integration.kafka.support.KafkaConsumerContext.class); - Assert.assertNotNull(consumerContext2); - } - - @SuppressWarnings("unchecked") - @Test - public void testConsumerContextConfigurations() { - final org.springframework.integration.kafka.support.KafkaConsumerContext consumerContext = - appContext.getBean("consumerContext1", - org.springframework.integration.kafka.support.KafkaConsumerContext.class); - Assert.assertNotNull(consumerContext); - final org.springframework.integration.kafka.support.ConsumerConfiguration cc = - consumerContext.getConsumerConfiguration("default1"); - final org.springframework.integration.kafka.support.ConsumerMetadata cm = cc.getConsumerMetadata(); - Assert.assertTrue(cm.getTopicStreamMap().get("test1") == 3); - Assert.assertTrue(cm.getTopicStreamMap().get("test2") == 4); - Assert.assertNotNull(cm); - final org.springframework.integration.kafka.support.ConsumerConfiguration cc2 = - consumerContext.getConsumerConfiguration("default2"); - final org.springframework.integration.kafka.support.ConsumerMetadata cm2 = - cc2.getConsumerMetadata(); - Assert.assertTrue(cm2.getTopicStreamMap().get("test3") == 1); - Assert.assertNotNull(cm2); - final org.springframework.integration.kafka.support.KafkaConsumerContext consumerContext2 = - appContext.getBean("consumerContext2", - org.springframework.integration.kafka.support.KafkaConsumerContext.class); - Assert.assertNotNull(consumerContext2); - final org.springframework.integration.kafka.support.ConsumerConfiguration otherCC = - consumerContext2.getConsumerConfiguration("default1"); - final org.springframework.integration.kafka.support.ConsumerMetadata otherCM = - otherCC.getConsumerMetadata(); - Assert.assertTrue(otherCM.getTopicStreamMap().get("test4") == 3); - } -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaOutboundAdapterParserTests-context.xml b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaOutboundAdapterParserTests-context.xml index 24bbfd3fa5..941a86d1d3 100644 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaOutboundAdapterParserTests-context.xml +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaOutboundAdapterParserTests-context.xml @@ -9,58 +9,31 @@ http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd http://www.springframework.org/schema/task http://www.springframework.org/schema/task/spring-task.xsd"> - - - + - - - - - - - - - + + + + + + + + + + - - - - - - - diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaOutboundAdapterParserTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaOutboundAdapterParserTests.java index a6ce36aaa8..48b548fa3d 100644 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaOutboundAdapterParserTests.java +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaOutboundAdapterParserTests.java @@ -16,29 +16,16 @@ package org.springframework.integration.kafka.config.xml; -import static org.hamcrest.Matchers.instanceOf; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; -import org.junit.ClassRule; import org.junit.Test; import org.junit.runner.RunWith; -import org.springframework.aop.Advisor; -import org.springframework.aop.framework.Advised; -import org.springframework.aop.support.AopUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.context.ApplicationContext; -import org.springframework.integration.endpoint.PollingConsumer; -import org.springframework.integration.handler.advice.RequestHandlerCircuitBreakerAdvice; import org.springframework.integration.kafka.outbound.KafkaProducerMessageHandler; -import org.springframework.integration.kafka.rule.KafkaEmbedded; -import org.springframework.integration.kafka.rule.KafkaRule; -import org.springframework.integration.kafka.support.KafkaProducerContext; import org.springframework.integration.test.util.TestUtils; -import org.springframework.messaging.MessageHandler; import org.springframework.test.annotation.DirtiesContext; import org.springframework.test.context.ContextConfiguration; import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; @@ -54,38 +41,19 @@ import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; @DirtiesContext public class KafkaOutboundAdapterParserTests { - @ClassRule - public static KafkaRule kafkaRunning = new KafkaEmbedded(1); - @Autowired private ApplicationContext appContext; @Test public void testOutboundAdapterConfiguration() { - PollingConsumer pollingConsumer = this.appContext.getBean("kafkaOutboundChannelAdapter", PollingConsumer.class); - KafkaProducerMessageHandler messageHandler - = this.appContext.getBean("org.springframework.integration.kafka.outbound.KafkaProducerMessageHandler#0", - KafkaProducerMessageHandler.class); - assertNotNull(pollingConsumer); + KafkaProducerMessageHandler messageHandler + = this.appContext.getBean("kafkaOutboundChannelAdapter.handler", KafkaProducerMessageHandler.class); assertNotNull(messageHandler); assertEquals(messageHandler.getOrder(), 3); assertEquals("foo", TestUtils.getPropertyValue(messageHandler, "topicExpression.literalValue")); assertEquals("'bar'", TestUtils.getPropertyValue(messageHandler, "messageKeyExpression.expression")); assertEquals("2", TestUtils.getPropertyValue(messageHandler, "partitionIdExpression.expression")); assertEquals(true, TestUtils.getPropertyValue(messageHandler, "enableHeaderRouting")); - KafkaProducerContext producerContext = messageHandler.getKafkaProducerContext(); - assertNotNull(producerContext); - assertEquals(producerContext.getProducerConfigurations().size(), 2); - - KafkaProducerMessageHandler messageHandler2 - = this.appContext.getBean("org.springframework.integration.kafka.outbound.KafkaProducerMessageHandler#1", - KafkaProducerMessageHandler.class); - assertEquals(false, TestUtils.getPropertyValue(messageHandler2, "enableHeaderRouting")); - MessageHandler advisedHandler = TestUtils.getPropertyValue(pollingConsumer, "handler", MessageHandler.class); - assertTrue(AopUtils.isJdkDynamicProxy(advisedHandler)); - Advisor[] advisors = ((Advised) advisedHandler).getAdvisors(); - assertEquals(1, advisors.length); - assertThat(advisors[0].getAdvice(), instanceOf(RequestHandlerCircuitBreakerAdvice.class)); } } diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaProducerContextParserTests-context.xml b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaProducerContextParserTests-context.xml deleted file mode 100644 index 7019857e82..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaProducerContextParserTests-context.xml +++ /dev/null @@ -1,64 +0,0 @@ - - - - - - - - 3600000 - 5 - 5242880 - - - - - - localhost:9092 - localhost:9091 - test1 - test2 - - - - - - - - - - - - - - - - - - - - - - diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaProducerContextParserTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaProducerContextParserTests.java deleted file mode 100644 index ff68d8d705..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/KafkaProducerContextParserTests.java +++ /dev/null @@ -1,143 +0,0 @@ -/* - * Copyright 2013-2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.kafka.config.xml; - -import static org.hamcrest.Matchers.equalTo; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertSame; - -import java.util.Map; - -import org.apache.kafka.clients.producer.Producer; -import org.apache.kafka.common.serialization.Serializer; -import org.junit.Assert; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.runner.RunWith; - -import org.springframework.beans.DirectFieldAccessor; -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; -import org.springframework.core.convert.ConversionService; -import org.springframework.core.convert.TypeDescriptor; -import org.springframework.integration.kafka.rule.KafkaEmbedded; -import org.springframework.integration.kafka.rule.KafkaRule; -import org.springframework.integration.kafka.support.KafkaProducerContext; -import org.springframework.integration.kafka.support.ProducerConfiguration; -import org.springframework.integration.kafka.support.ProducerListener; -import org.springframework.integration.kafka.support.ProducerMetadata; -import org.springframework.integration.kafka.util.EncoderAdaptingSerializer; -import org.springframework.integration.test.util.TestUtils; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; - -import kafka.serializer.Encoder; - -/** - * @author Soby Chacko - * @author Gary Russell - * @since 0.5 - */ -@RunWith(SpringJUnit4ClassRunner.class) -@ContextConfiguration -public class KafkaProducerContextParserTests { - - @ClassRule - public static KafkaRule kafkaRule = new KafkaEmbedded(1); - - @Autowired - private ApplicationContext appContext; - - @Test - @SuppressWarnings({"unchecked","rawtypes"}) - public void testProducerContextConfiguration(){ - final KafkaProducerContext producerContext = appContext.getBean("producerContext", KafkaProducerContext.class); - Assert.assertNotNull(producerContext); - - final Map> producerConfigurations = producerContext.getProducerConfigurations(); - assertEquals(producerConfigurations.size(), 2); - - final ProducerConfiguration producerConfigurationTest1 = producerConfigurations.get("test1"); - Assert.assertNotNull(producerConfigurationTest1); - final ProducerMetadata producerMetadataTest1 = producerConfigurationTest1.getProducerMetadata(); - assertEquals(producerMetadataTest1.getTopic(), "test1"); - assertEquals(producerMetadataTest1.getCompressionType(), ProducerMetadata.CompressionType.none); - assertEquals(producerMetadataTest1.getKeyClassType(), java.lang.String.class); - assertEquals(producerMetadataTest1.getValueClassType(), java.lang.String.class); - - final Encoder valueEncoder = appContext.getBean("valueEncoder", Encoder.class); - Assert.assertThat((Class) producerMetadataTest1.getKeySerializer().getClass(), equalTo((Class) EncoderAdaptingSerializer.class)); - Assert.assertThat(((EncoderAdaptingSerializer) producerMetadataTest1.getKeySerializer()).getEncoder(), equalTo((Encoder) valueEncoder)); - Assert.assertThat((Class)producerMetadataTest1.getValueSerializer().getClass(), equalTo((Class)EncoderAdaptingSerializer.class)); - Assert.assertThat(((EncoderAdaptingSerializer)producerMetadataTest1.getValueSerializer()).getEncoder(), equalTo((Encoder)valueEncoder)); - - DirectFieldAccessor directFieldAccessor = new DirectFieldAccessor(producerConfigurationTest1); - Producer producerTest1 = (Producer) directFieldAccessor.getPropertyValue("producer"); - assertEquals(producerConfigurationTest1.getProducerMetadata(), producerMetadataTest1); - - final ProducerConfiguration producerConfigurationTest2 = producerConfigurations.get("test2"); - Assert.assertNotNull(producerConfigurationTest2); - final ProducerMetadata producerMetadataTest2 = producerConfigurationTest2.getProducerMetadata(); - assertEquals(producerMetadataTest2.getTopic(), "test2"); - assertEquals(producerMetadataTest2.getCompressionType(), ProducerMetadata.CompressionType.none); - - DirectFieldAccessor directFieldAccessor2 = new DirectFieldAccessor(producerConfigurationTest2); - Producer producerTest2 = (Producer) directFieldAccessor2.getPropertyValue("producer"); - assertEquals(producerConfigurationTest2.getProducerMetadata(), producerMetadataTest2); - - final Serializer stringSerializer = appContext.getBean("stringSerializer", Serializer.class); - assertSame(stringSerializer, producerConfigurationTest2.getProducerMetadata().getKeySerializer()); - assertSame(stringSerializer, producerConfigurationTest2.getProducerMetadata().getValueSerializer()); - - final ConversionService conversionService = appContext.getBean("conversionService", ConversionService.class); - ConversionService configuredConversionService = (ConversionService) directFieldAccessor2.getPropertyValue("conversionService"); - assertSame(conversionService, configuredConversionService); - - final ProducerListener producerListener = appContext.getBean("producerListener", ProducerListener.class); - ProducerListener configuredProducerListener = (ProducerListener) directFieldAccessor2.getPropertyValue("producerListener"); - assertSame(producerListener, configuredProducerListener); - - assertEquals(9876,producerConfigurationTest2.getProducerMetadata().getBatchBytes()); - - assertFalse(TestUtils.getPropertyValue(producerContext, "autoStartup", Boolean.class)); - assertEquals(123, TestUtils.getPropertyValue(producerContext, "phase")); - - assertEquals("windows-1251", producerConfigurationTest2.getProducerMetadata().getCharset().name()); - } - - public static class StubConversionService implements ConversionService { - @Override - public boolean canConvert(Class sourceType, Class targetType) { - return false; - } - - @Override - public boolean canConvert(TypeDescriptor sourceType, TypeDescriptor targetType) { - return false; - } - - @Override - public T convert(Object source, Class targetType) { - return null; - } - - @Override - public Object convert(Object source, TypeDescriptor sourceType, TypeDescriptor targetType) { - return null; - } - } -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/ZookeeperConnectParserTests-context.xml b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/ZookeeperConnectParserTests-context.xml deleted file mode 100644 index f5d07846a6..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/ZookeeperConnectParserTests-context.xml +++ /dev/null @@ -1,14 +0,0 @@ - - - - - - - diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/ZookeeperConnectParserTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/ZookeeperConnectParserTests.java deleted file mode 100644 index 37adf3bf04..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/ZookeeperConnectParserTests.java +++ /dev/null @@ -1,65 +0,0 @@ -/* - * Copyright 2013-2014 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.kafka.config.xml; - -import org.junit.Assert; -import org.junit.ClassRule; -import org.junit.Test; -import org.junit.runner.RunWith; - -import org.springframework.beans.factory.annotation.Autowired; -import org.springframework.context.ApplicationContext; -import org.springframework.integration.kafka.core.ZookeeperConnectDefaults; -import org.springframework.integration.kafka.rule.KafkaRunning; -import org.springframework.integration.kafka.support.ZookeeperConnect; -import org.springframework.test.context.ContextConfiguration; -import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; - -/** - * @author Soby Chacko - * @author Gary Russell - * @since 0.5 - */ -@RunWith(SpringJUnit4ClassRunner.class) -@ContextConfiguration -public class ZookeeperConnectParserTests { - - @ClassRule - public static KafkaRunning kafkaRunning = KafkaRunning.isRunning(); - - @Autowired - private ApplicationContext appContext; - - @Test - public void testCustomKafkaBrokerConfiguration() { - final ZookeeperConnect broker = appContext.getBean("zookeeperConnect", ZookeeperConnect.class); - - Assert.assertEquals("localhost:2181", broker.getZkConnect()); - Assert.assertEquals("10000", broker.getZkConnectionTimeout()); - Assert.assertEquals("10000", broker.getZkSessionTimeout()); - Assert.assertEquals("200", broker.getZkSyncTime()); - } - - @Test - public void testDefaultKafkaBrokerConfiguration() { - final ZookeeperConnect broker = appContext.getBean("defaultZookeeperConnect", ZookeeperConnect.class); - - Assert.assertEquals(ZookeeperConnectDefaults.ZK_CONNECT, broker.getZkConnect()); - Assert.assertEquals(ZookeeperConnectDefaults.ZK_CONNECTION_TIMEOUT, broker.getZkConnectionTimeout()); - Assert.assertEquals(ZookeeperConnectDefaults.ZK_SESSION_TIMEOUT, broker.getZkSessionTimeout()); - Assert.assertEquals(ZookeeperConnectDefaults.ZK_SYNC_TIME, broker.getZkSyncTime()); - } -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/kafkaInboundAdapterCommon-context.xml b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/kafkaInboundAdapterCommon-context.xml deleted file mode 100644 index 42e907ef89..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/config/xml/kafkaInboundAdapterCommon-context.xml +++ /dev/null @@ -1,18 +0,0 @@ - - - - - - - - - - diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/core/MetadataCacheTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/core/MetadataCacheTests.java deleted file mode 100644 index 44ae27d1fd..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/core/MetadataCacheTests.java +++ /dev/null @@ -1,262 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.core; - -import static org.hamcrest.CoreMatchers.equalTo; -import static org.hamcrest.CoreMatchers.hasItem; -import static org.hamcrest.CoreMatchers.notNullValue; -import static org.hamcrest.CoreMatchers.nullValue; -import static org.hamcrest.collection.IsCollectionWithSize.hasSize; -import static org.junit.Assert.assertThat; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; - -import org.junit.Test; - -import kafka.api.PartitionMetadata; -import kafka.cluster.Broker; -import kafka.common.ErrorMapping; -import kafka.javaapi.TopicMetadata; -import scala.Some; -import scala.collection.Seq; -import scala.collection.mutable.HashSet; -import scala.collection.mutable.HashSet$; - -/** - * @author Marius Bogoevici - */ -public class MetadataCacheTests { - - - public static final String NONEXISTENT_TOPIC = "nonexistent"; - - public static final String EXISTING_TOPIC = "existing"; - - public static final String EXISTING_TOPIC_2 = "existing2"; - - @Test - public void testMetadataCacheEmpty() throws Exception { - MetadataCache metadataCache = new MetadataCache(Collections.emptySet()); - - // blank metadatacache is empty - assertThat(metadataCache.getLeader(new Partition(NONEXISTENT_TOPIC, 0)), nullValue()); - assertThat(metadataCache.getPartitions(NONEXISTENT_TOPIC), nullValue()); - } - - @Test - public void testMetadataCacheInitialized() throws Exception { - ArrayList topicMetadatas = new ArrayList(); - HashSet partitionMetadatas = HashSet$.MODULE$.empty(); - partitionMetadatas.$plus$eq(new PartitionMetadata(0, asScalaOption(new Broker(0, "host0", 1000)), - emptySeq(), emptySeq(), ErrorMapping.NoError())); - partitionMetadatas.$plus$eq(new PartitionMetadata(1, asScalaOption(new Broker(0, "host1", 1001)), - emptySeq(), emptySeq(), ErrorMapping.NoError())); - topicMetadatas.add(new TopicMetadata(new kafka.api.TopicMetadata(EXISTING_TOPIC, partitionMetadatas.toSeq(), - ErrorMapping.NoError()))); - MetadataCache metadataCache = new MetadataCache(topicMetadatas); - - // Initializing MetadataCache with some topic metadatas will find the data in - assertThat(metadataCache.getLeader(new Partition(NONEXISTENT_TOPIC, 0)), nullValue()); - assertThat(metadataCache.getPartitions(NONEXISTENT_TOPIC), nullValue()); - assertThat(metadataCache.getPartitions(EXISTING_TOPIC), hasSize(2)); - assertThat(metadataCache.getPartitions(EXISTING_TOPIC), hasItem(new Partition(EXISTING_TOPIC, 0))); - assertThat(metadataCache.getPartitions(EXISTING_TOPIC), hasItem(new Partition(EXISTING_TOPIC, 1))); - assertThat(metadataCache.getLeader(new Partition(EXISTING_TOPIC, 0)), notNullValue()); - assertThat(metadataCache.getLeader(new Partition(EXISTING_TOPIC, 0)).getHost(), equalTo("host0")); - assertThat(metadataCache.getLeader(new Partition(EXISTING_TOPIC, 0)).getPort(), equalTo(1000)); - assertThat(metadataCache.getLeader(new Partition(EXISTING_TOPIC, 1)), notNullValue()); - assertThat(metadataCache.getLeader(new Partition(EXISTING_TOPIC, 1)).getHost(), equalTo("host1")); - assertThat(metadataCache.getLeader(new Partition(EXISTING_TOPIC, 1)).getPort(), equalTo(1001)); - } - - @Test - public void testMetadataCacheMerged() throws Exception { - ArrayList topicMetadatas = new ArrayList(); - HashSet partitionMetadatas = HashSet$.MODULE$.empty(); - partitionMetadatas.$plus$eq(new PartitionMetadata(0, asScalaOption(new Broker(0, "host0", 1000)), emptySeq(), - emptySeq(), ErrorMapping.NoError())); - partitionMetadatas.$plus$eq(new PartitionMetadata(1, asScalaOption(new Broker(0, "host1", 1001)), emptySeq(), - emptySeq(), ErrorMapping.NoError())); - topicMetadatas.add(new TopicMetadata(new kafka.api.TopicMetadata(EXISTING_TOPIC, partitionMetadatas.toSeq(), - ErrorMapping.NoError()))); - - MetadataCache metadataCache = new MetadataCache(Collections.emptySet()); - - assertThat(metadataCache.getLeader(new Partition(NONEXISTENT_TOPIC, 0)), nullValue()); - assertThat(metadataCache.getPartitions(NONEXISTENT_TOPIC), nullValue()); - assertThat(metadataCache.getLeader(new Partition(EXISTING_TOPIC, 0)), nullValue()); - assertThat(metadataCache.getPartitions(EXISTING_TOPIC), nullValue()); - - - MetadataCache newMetadataCache = metadataCache.merge(topicMetadatas); - - // merging MetadataCache with new topic metadata results in an object that also has the new data - assertThat(newMetadataCache.getLeader(new Partition(NONEXISTENT_TOPIC, 0)), nullValue()); - assertThat(newMetadataCache.getPartitions(NONEXISTENT_TOPIC), nullValue()); - assertThat(newMetadataCache.getPartitions(EXISTING_TOPIC), hasSize(2)); - assertThat(newMetadataCache.getPartitions(EXISTING_TOPIC), hasItem(new Partition(EXISTING_TOPIC, 0))); - assertThat(newMetadataCache.getPartitions(EXISTING_TOPIC), hasItem(new Partition(EXISTING_TOPIC, 1))); - assertThat(newMetadataCache.getLeader(new Partition(EXISTING_TOPIC, 0)), notNullValue()); - assertThat(newMetadataCache.getLeader(new Partition(EXISTING_TOPIC, 0)).getHost(), equalTo("host0")); - assertThat(newMetadataCache.getLeader(new Partition(EXISTING_TOPIC, 0)).getPort(), equalTo(1000)); - assertThat(newMetadataCache.getLeader(new Partition(EXISTING_TOPIC, 1)), notNullValue()); - assertThat(newMetadataCache.getLeader(new Partition(EXISTING_TOPIC, 1)).getHost(), equalTo("host1")); - assertThat(newMetadataCache.getLeader(new Partition(EXISTING_TOPIC, 1)).getPort(), equalTo(1001)); - } - - @Test - public void testMetadataCacheMergedWithOverride() throws Exception { - List topicMetadatas = new ArrayList(); - HashSet partitionMetadatas = HashSet$.MODULE$.empty(); - partitionMetadatas.$plus$eq(new PartitionMetadata(0, asScalaOption(new Broker(0, "host0", 1000)), emptySeq(), - emptySeq(), ErrorMapping.NoError())); - partitionMetadatas.$plus$eq(new PartitionMetadata(1, asScalaOption(new Broker(0, "host1", 1001)), emptySeq(), - emptySeq(), ErrorMapping.NoError())); - topicMetadatas.add(new TopicMetadata(new kafka.api.TopicMetadata(EXISTING_TOPIC, partitionMetadatas.toSeq(), - ErrorMapping.NoError()))); - - MetadataCache metadataCache = new MetadataCache(Collections.emptySet()); - - //empty MetadataCache has no data - assertThat(metadataCache.getLeader(new Partition(NONEXISTENT_TOPIC, 0)), nullValue()); - assertThat(metadataCache.getPartitions(NONEXISTENT_TOPIC), nullValue()); - assertThat(metadataCache.getLeader(new Partition(EXISTING_TOPIC, 0)), nullValue()); - assertThat(metadataCache.getPartitions(EXISTING_TOPIC), nullValue()); - assertThat(metadataCache.getLeader(new Partition(EXISTING_TOPIC_2, 0)), nullValue()); - assertThat(metadataCache.getPartitions(EXISTING_TOPIC_2), nullValue()); - - MetadataCache metadataCacheWithTopic1 = metadataCache.merge(topicMetadatas); - - // merging MetadataCache with new topic metadata results in an object that also has the new data - assertThat(metadataCacheWithTopic1.getLeader(new Partition(NONEXISTENT_TOPIC, 0)), nullValue()); - assertThat(metadataCacheWithTopic1.getPartitions(NONEXISTENT_TOPIC), nullValue()); - assertThat(metadataCacheWithTopic1.getPartitions(EXISTING_TOPIC), hasSize(2)); - assertThat(metadataCacheWithTopic1.getPartitions(EXISTING_TOPIC), hasItem(new Partition(EXISTING_TOPIC, 0))); - assertThat(metadataCacheWithTopic1.getPartitions(EXISTING_TOPIC), hasItem(new Partition(EXISTING_TOPIC, 1))); - assertThat(metadataCacheWithTopic1.getLeader(new Partition(EXISTING_TOPIC, 0)), notNullValue()); - assertThat(metadataCacheWithTopic1.getLeader(new Partition(EXISTING_TOPIC, 0)).getHost(), equalTo("host0")); - assertThat(metadataCacheWithTopic1.getLeader(new Partition(EXISTING_TOPIC, 0)).getPort(), equalTo(1000)); - assertThat(metadataCacheWithTopic1.getLeader(new Partition(EXISTING_TOPIC, 1)), notNullValue()); - assertThat(metadataCacheWithTopic1.getLeader(new Partition(EXISTING_TOPIC, 1)).getHost(), equalTo("host1")); - assertThat(metadataCacheWithTopic1.getLeader(new Partition(EXISTING_TOPIC, 1)).getPort(), equalTo(1001)); - assertThat(metadataCacheWithTopic1.getLeader(new Partition(EXISTING_TOPIC, 2)), nullValue()); - - ArrayList topic2Metadatas = new ArrayList(); - HashSet topic2partitionMetadatas = HashSet$.MODULE$.empty(); - topic2partitionMetadatas.$plus$eq(new PartitionMetadata(0, asScalaOption(new Broker(0, "host1", 1002)), - emptySeq(), emptySeq(), ErrorMapping.NoError())); - topic2partitionMetadatas.$plus$eq(new PartitionMetadata(1, asScalaOption(new Broker(0, "host2", 1003)), - emptySeq(), emptySeq(), ErrorMapping.NoError())); - topic2partitionMetadatas.$plus$eq(new PartitionMetadata(2, asScalaOption(new Broker(0, "host3", 1004)), emptySeq(), - emptySeq(), ErrorMapping.NoError())); - topic2Metadatas.add(new TopicMetadata(new kafka.api.TopicMetadata(EXISTING_TOPIC_2, topic2partitionMetadatas.toSeq(), - ErrorMapping.NoError()))); - MetadataCache metadataCacheWithTopic2 = metadataCacheWithTopic1.merge(topic2Metadatas); - - - assertThat(metadataCacheWithTopic2.getLeader(new Partition(NONEXISTENT_TOPIC, 0)), nullValue()); - assertThat(metadataCacheWithTopic2.getPartitions(NONEXISTENT_TOPIC), nullValue()); - assertThat(metadataCacheWithTopic2.getPartitions(EXISTING_TOPIC), hasSize(2)); - assertThat(metadataCacheWithTopic2.getPartitions(EXISTING_TOPIC), hasItem(new Partition(EXISTING_TOPIC, 0))); - assertThat(metadataCacheWithTopic2.getPartitions(EXISTING_TOPIC), hasItem(new Partition(EXISTING_TOPIC, 1))); - assertThat(metadataCacheWithTopic2.getLeader(new Partition(EXISTING_TOPIC, 0)), notNullValue()); - assertThat(metadataCacheWithTopic2.getLeader(new Partition(EXISTING_TOPIC, 0)).getHost(), equalTo("host0")); - assertThat(metadataCacheWithTopic2.getLeader(new Partition(EXISTING_TOPIC, 0)).getPort(), equalTo(1000)); - assertThat(metadataCacheWithTopic2.getLeader(new Partition(EXISTING_TOPIC, 1)), notNullValue()); - assertThat(metadataCacheWithTopic2.getLeader(new Partition(EXISTING_TOPIC, 1)).getHost(), equalTo("host1")); - assertThat(metadataCacheWithTopic2.getLeader(new Partition(EXISTING_TOPIC, 1)).getPort(), equalTo(1001)); - assertThat(metadataCacheWithTopic2.getLeader(new Partition(EXISTING_TOPIC, 2)), nullValue()); - assertThat(metadataCacheWithTopic2.getPartitions(EXISTING_TOPIC_2), notNullValue()); - assertThat(metadataCacheWithTopic2.getPartitions(EXISTING_TOPIC_2), hasSize(3)); - assertThat(metadataCacheWithTopic2.getPartitions(EXISTING_TOPIC_2), hasItem(new Partition(EXISTING_TOPIC_2, 0))); - assertThat(metadataCacheWithTopic2.getPartitions(EXISTING_TOPIC_2), hasItem(new Partition(EXISTING_TOPIC_2, 1))); - assertThat(metadataCacheWithTopic2.getPartitions(EXISTING_TOPIC_2), hasItem(new Partition(EXISTING_TOPIC_2, 2))); - assertThat(metadataCacheWithTopic2.getLeader(new Partition(EXISTING_TOPIC_2, 0)), notNullValue()); - assertThat(metadataCacheWithTopic2.getLeader(new Partition(EXISTING_TOPIC_2, 0)).getHost(), equalTo("host1")); - assertThat(metadataCacheWithTopic2.getLeader(new Partition(EXISTING_TOPIC_2, 0)).getPort(), equalTo(1002)); - assertThat(metadataCacheWithTopic2.getLeader(new Partition(EXISTING_TOPIC_2, 1)), notNullValue()); - assertThat(metadataCacheWithTopic2.getLeader(new Partition(EXISTING_TOPIC_2, 1)).getHost(), equalTo("host2")); - assertThat(metadataCacheWithTopic2.getLeader(new Partition(EXISTING_TOPIC_2, 1)).getPort(), equalTo(1003)); - assertThat(metadataCacheWithTopic2.getLeader(new Partition(EXISTING_TOPIC_2, 2)), notNullValue()); - assertThat(metadataCacheWithTopic2.getLeader(new Partition(EXISTING_TOPIC_2, 2)).getHost(), equalTo("host3")); - assertThat(metadataCacheWithTopic2.getLeader(new Partition(EXISTING_TOPIC_2, 2)).getPort(), equalTo(1004)); - assertThat(metadataCacheWithTopic2.getLeader(new Partition(EXISTING_TOPIC_2, 3)), nullValue()); - - // merging overrides data - List overridingTopicMetadatas = new ArrayList(); - HashSet overridingPartitionsMetadatas = HashSet$.MODULE$.empty(); - overridingPartitionsMetadatas.$plus$eq(new PartitionMetadata(0, asScalaOption(new Broker(0, "host10", 2000)), - emptySeq(), emptySeq(), ErrorMapping.NoError())); - overridingPartitionsMetadatas.$plus$eq(new PartitionMetadata(1, asScalaOption(new Broker(0, "host11", 2001)), - emptySeq(), emptySeq(), ErrorMapping.NoError())); - overridingPartitionsMetadatas.$plus$eq(new PartitionMetadata(2, asScalaOption(new Broker(0, "host12", 2002)), - emptySeq(), emptySeq(), ErrorMapping.NoError())); - overridingPartitionsMetadatas.$plus$eq(new PartitionMetadata(3, asScalaOption(new Broker(0, "host13", 2003)), - emptySeq(), emptySeq(), ErrorMapping.NoError())); - overridingTopicMetadatas.add(new TopicMetadata(new kafka.api.TopicMetadata(EXISTING_TOPIC, - overridingPartitionsMetadatas.toSeq(), ErrorMapping.NoError()))); - - MetadataCache metadataCacheOverridden = metadataCacheWithTopic2.merge(overridingTopicMetadatas); - - assertThat(metadataCacheOverridden.getLeader(new Partition(NONEXISTENT_TOPIC, 0)), nullValue()); - assertThat(metadataCacheOverridden.getPartitions(NONEXISTENT_TOPIC), nullValue()); - assertThat(metadataCacheOverridden.getPartitions(EXISTING_TOPIC), hasSize(4)); - assertThat(metadataCacheOverridden.getPartitions(EXISTING_TOPIC), hasItem(new Partition(EXISTING_TOPIC, 0))); - assertThat(metadataCacheOverridden.getPartitions(EXISTING_TOPIC), hasItem(new Partition(EXISTING_TOPIC, 1))); - assertThat(metadataCacheOverridden.getPartitions(EXISTING_TOPIC), hasItem(new Partition(EXISTING_TOPIC, 2))); - assertThat(metadataCacheOverridden.getPartitions(EXISTING_TOPIC), hasItem(new Partition(EXISTING_TOPIC, 3))); - assertThat(metadataCacheOverridden.getLeader(new Partition(EXISTING_TOPIC, 0)), notNullValue()); - assertThat(metadataCacheOverridden.getLeader(new Partition(EXISTING_TOPIC, 0)).getHost(), equalTo("host10")); - assertThat(metadataCacheOverridden.getLeader(new Partition(EXISTING_TOPIC, 0)).getPort(), equalTo(2000)); - assertThat(metadataCacheOverridden.getLeader(new Partition(EXISTING_TOPIC, 1)), notNullValue()); - assertThat(metadataCacheOverridden.getLeader(new Partition(EXISTING_TOPIC, 1)).getHost(), equalTo("host11")); - assertThat(metadataCacheOverridden.getLeader(new Partition(EXISTING_TOPIC, 1)).getPort(), equalTo(2001)); - assertThat(metadataCacheOverridden.getLeader(new Partition(EXISTING_TOPIC, 2)), notNullValue()); - assertThat(metadataCacheOverridden.getLeader(new Partition(EXISTING_TOPIC, 2)).getHost(), equalTo("host12")); - assertThat(metadataCacheOverridden.getLeader(new Partition(EXISTING_TOPIC, 2)).getPort(), equalTo(2002)); - assertThat(metadataCacheOverridden.getLeader(new Partition(EXISTING_TOPIC, 3)), notNullValue()); - assertThat(metadataCacheOverridden.getLeader(new Partition(EXISTING_TOPIC, 3)).getHost(), equalTo("host13")); - assertThat(metadataCacheOverridden.getLeader(new Partition(EXISTING_TOPIC, 3)).getPort(), equalTo(2003)); - assertThat(metadataCacheOverridden.getLeader(new Partition(EXISTING_TOPIC, 4)), nullValue()); - assertThat(metadataCacheOverridden.getPartitions(EXISTING_TOPIC_2), hasSize(3)); - assertThat(metadataCacheOverridden.getPartitions(EXISTING_TOPIC_2), hasItem(new Partition(EXISTING_TOPIC_2, 0))); - assertThat(metadataCacheOverridden.getPartitions(EXISTING_TOPIC_2), hasItem(new Partition(EXISTING_TOPIC_2, 1))); - assertThat(metadataCacheOverridden.getPartitions(EXISTING_TOPIC_2), hasItem(new Partition(EXISTING_TOPIC_2, 2))); - assertThat(metadataCacheOverridden.getLeader(new Partition(EXISTING_TOPIC_2, 0)), notNullValue()); - assertThat(metadataCacheOverridden.getLeader(new Partition(EXISTING_TOPIC_2, 0)).getHost(), equalTo("host1")); - assertThat(metadataCacheOverridden.getLeader(new Partition(EXISTING_TOPIC_2, 0)).getPort(), equalTo(1002)); - assertThat(metadataCacheOverridden.getLeader(new Partition(EXISTING_TOPIC_2, 1)), notNullValue()); - assertThat(metadataCacheOverridden.getLeader(new Partition(EXISTING_TOPIC_2, 1)).getHost(), equalTo("host2")); - assertThat(metadataCacheOverridden.getLeader(new Partition(EXISTING_TOPIC_2, 1)).getPort(), equalTo(1003)); - assertThat(metadataCacheOverridden.getLeader(new Partition(EXISTING_TOPIC_2, 2)), notNullValue()); - assertThat(metadataCacheOverridden.getLeader(new Partition(EXISTING_TOPIC_2, 2)).getHost(), equalTo("host3")); - assertThat(metadataCacheOverridden.getLeader(new Partition(EXISTING_TOPIC_2, 2)).getPort(), equalTo(1004)); - assertThat(metadataCacheOverridden.getLeader(new Partition(EXISTING_TOPIC_2, 3)), nullValue()); - - } - - public Seq emptySeq() { - return HashSet$.MODULE$.empty().toSeq(); - } - - public Some asScalaOption(T object) { - return new Some(object); - } - -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/inbound/MessageDrivenAdapterTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/inbound/MessageDrivenAdapterTests.java new file mode 100644 index 0000000000..59144271dc --- /dev/null +++ b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/inbound/MessageDrivenAdapterTests.java @@ -0,0 +1,75 @@ +/* + * Copyright 2016 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.kafka.inbound; + +import static org.junit.Assert.assertNotNull; + +import java.util.Map; + +import org.apache.kafka.clients.consumer.ConsumerConfig; +import org.junit.ClassRule; +import org.junit.Test; + +import org.springframework.integration.channel.QueueChannel; +import org.springframework.kafka.core.DefaultKafkaConsumerFactory; +import org.springframework.kafka.core.DefaultKafkaProducerFactory; +import org.springframework.kafka.core.KafkaTemplate; +import org.springframework.kafka.core.ProducerFactory; +import org.springframework.kafka.listener.KafkaMessageListenerContainer; +import org.springframework.kafka.test.rule.KafkaEmbedded; +import org.springframework.kafka.test.utils.KafkaTestUtils; +import org.springframework.messaging.Message; + +/** + * @author Gary Russell + * @author Artem Bilan + * @since 2.0 + * + */ +public class MessageDrivenAdapterTests { + + private static String topic1 = "testTopic1"; + + private static String topic2 = "testTopic2"; + + @ClassRule + public static KafkaEmbedded embeddedKafka = new KafkaEmbedded(1, true, topic1, topic2); + + @Test + public void testInbound() throws Exception { + Map props = KafkaTestUtils.consumerProps("test1", "true", embeddedKafka); + props.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest"); + DefaultKafkaConsumerFactory cf = new DefaultKafkaConsumerFactory(props); + KafkaMessageListenerContainer container = + new KafkaMessageListenerContainer<>(cf, topic1); + KafkaMessageDrivenChannelAdapter adapter = new KafkaMessageDrivenChannelAdapter<>(container); + QueueChannel out = new QueueChannel(); + adapter.setOutputChannel(out); + adapter.afterPropertiesSet(); + adapter.start(); + + Map senderProps = KafkaTestUtils.producerProps(embeddedKafka); + ProducerFactory pf = new DefaultKafkaProducerFactory(senderProps); + KafkaTemplate template = new KafkaTemplate<>(pf); + template.setDefaultTopic("testTopic1"); + template.convertAndSend("foo"); + Message received = out.receive(10000); + assertNotNull(received); + adapter.stop(); + } + +} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/AbstractBrokerTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/AbstractBrokerTests.java deleted file mode 100644 index 906189616f..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/AbstractBrokerTests.java +++ /dev/null @@ -1,220 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.listener; - -import static org.springframework.integration.kafka.util.TopicUtils.ensureTopicCreated; -import static scala.collection.JavaConversions.asScalaBuffer; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Properties; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import com.gs.collections.api.RichIterable; -import com.gs.collections.api.block.function.Function2; -import com.gs.collections.api.multimap.Multimap; -import com.gs.collections.api.multimap.MutableMultimap; -import com.gs.collections.api.tuple.Pair; -import com.gs.collections.impl.factory.Multimaps; -import com.gs.collections.impl.tuple.Tuples; -import kafka.admin.AdminUtils; -import kafka.utils.TestUtils; -import org.I0Itec.zkclient.ZkClient; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.clients.producer.KafkaProducer; -import org.apache.kafka.clients.producer.Producer; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.clients.producer.RecordMetadata; -import org.junit.After; -import org.junit.Assert; -import scala.collection.JavaConversions; -import scala.collection.Map; -import scala.collection.immutable.List$; -import scala.collection.immutable.Map$; -import scala.collection.immutable.Seq; - -import org.springframework.integration.kafka.core.BrokerAddressListConfiguration; -import org.springframework.integration.kafka.core.Configuration; -import org.springframework.integration.kafka.core.ConnectionFactory; -import org.springframework.integration.kafka.core.DefaultConnectionFactory; -import org.springframework.integration.kafka.rule.KafkaRule; -import org.springframework.integration.kafka.serializer.common.StringEncoder; -import org.springframework.integration.kafka.util.EncoderAdaptingSerializer; - -/** - * @author Marius Bogoevici - */ -public abstract class AbstractBrokerTests { - - private static final Log log = LogFactory.getLog(AbstractBrokerTests.class); - - public static final String TEST_TOPIC = "test-topic"; - - private static final long SEND_TIMEOUT = 10000; - - public abstract KafkaRule getKafkaRule(); - - @After - public void cleanUp() { - deleteTopic(TEST_TOPIC); - } - - public void createTopic(String topicName, int partitionCount, int brokers, int replication) { - createTopic(getKafkaRule().getZkClient(), topicName, partitionCount, brokers, replication); - } - - @SuppressWarnings("unchecked") - public void createTopic(ZkClient zkClient, String topicName, int partitionCount, int brokers, int replication) { - MutableMultimap partitionDistribution = - createPartitionDistribution(partitionCount, brokers, replication); - ensureTopicCreated(zkClient, topicName, partitionCount, new Properties(), - toKafkaPartitionMap(partitionDistribution)); - } - - public void deleteTopic(String topicName) { - AdminUtils.deleteTopic(getKafkaRule().getZkClient(), topicName); - if (getKafkaRule().isEmbedded()) { - TestUtils.waitUntilMetadataIsPropagated(asScalaBuffer(getKafkaRule().getKafkaServers()), topicName, 0, 5000L); - } - else { - sleep(1000); - } - } - - public MutableMultimap createPartitionDistribution(int partitionCount, int brokers, - int replication) { - MutableMultimap partitionDistribution = Multimaps.mutable.list.with(); - for (int i = 0; i < partitionCount; i++) { - for (int j = 0; j < replication; j++) { - partitionDistribution.put(i, (i + j) % brokers); - } - } - return partitionDistribution; - } - - - public Configuration getKafkaConfiguration() { - BrokerAddressListConfiguration configuration = new BrokerAddressListConfiguration(getKafkaRule().getBrokerAddresses()); - configuration.setSocketTimeout(500); - return configuration; - } - - public static Collection> createMessages(int count, String topic, int partitionCount) { - return createMessagesInRange(0, count - 1, topic, partitionCount); - } - - public static Collection> createMessagesInRange(int start, int end, - String topic, int partitionCount) { - List> messages = new ArrayList<>(); - for (int i = start; i <= end; i++) { - messages.add(new ProducerRecord<>(topic, i % partitionCount, "Key " + i, "Message " + i)); - } - return messages; - } - - public Sender createMessageSender(String compression) { - Properties producerConfig = new Properties(); - producerConfig.setProperty("bootstrap.servers", getKafkaRule().getBrokersAsString()); - producerConfig.setProperty("compression.type", compression); - KafkaProducer producer = new KafkaProducer<>(producerConfig, - new EncoderAdaptingSerializer<>(new StringEncoder()), - new EncoderAdaptingSerializer<>(new StringEncoder())); - return new Sender<>(producer); - } - - public Sender createMessageSender(String compression, int brokerIndex) { - Properties producerConfig = new Properties(); - producerConfig.setProperty("bootstrap.servers", getKafkaRule().getBrokerAddresses()[brokerIndex].toString()); - producerConfig.setProperty("compression.type", compression); - KafkaProducer producer = new KafkaProducer<>(producerConfig, - new EncoderAdaptingSerializer<>(new StringEncoder()), - new EncoderAdaptingSerializer<>(new StringEncoder())); - return new Sender<>(producer); - } - - public ConnectionFactory getKafkaBrokerConnectionFactory() throws Exception { - DefaultConnectionFactory connectionFactory = new DefaultConnectionFactory(getKafkaConfiguration()); - connectionFactory.afterPropertiesSet(); - return connectionFactory; - } - - @SuppressWarnings({"rawtypes", "serial", "deprecation"}) - private Map toKafkaPartitionMap(Multimap partitions) { - java.util.Map> m = partitions.toMap() - .collect(new Function2, Pair>>() { - - @Override - public Pair> value(Integer argument1, RichIterable argument2) { - return Tuples.pair((Object) argument1, - List$.MODULE$.fromArray(argument2.toArray(new Object[0])).toSeq()); - } - - }); - return Map$.MODULE$.apply(JavaConversions.asScalaMap(m).toSeq()); - } - - private static void sleep(int time) { - try { - Thread.sleep(time); - } - catch (InterruptedException e) { - log.error(e); - } - } - - public class Sender { - - private Producer producer; - - public Sender(Producer producer) { - this.producer = producer; - } - - public void send(Collection> records) { - final CountDownLatch sendLatch = new CountDownLatch(records.size()); - final ArrayList exceptions = new ArrayList<>(); - for (ProducerRecord record : records) { - producer.send(record, new Callback() { - @Override - public void onCompletion(RecordMetadata metadata, Exception exception) { - sendLatch.countDown(); - if (exception != null) { - exceptions.add(exception); - } - } - }); - } - try { - sendLatch.await(SEND_TIMEOUT, TimeUnit.MILLISECONDS); - } - catch (InterruptedException e) { - throw new RuntimeException(e); - } - if (exceptions.size() > 0) { - for (Exception exception : exceptions) { - log.error("Error while sending messages:", exception); - } - Assert.fail("Cannot send messages"); - } - } - } - -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/AbstractMessageListenerContainerTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/AbstractMessageListenerContainerTests.java deleted file mode 100644 index ffd5c8c71b..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/AbstractMessageListenerContainerTests.java +++ /dev/null @@ -1,238 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package org.springframework.integration.kafka.listener; - -import static org.hamcrest.CoreMatchers.not; -import static org.hamcrest.collection.IsCollectionWithSize.hasSize; -import static org.hamcrest.collection.IsEmptyCollection.empty; -import static org.hamcrest.core.IsCollectionContaining.hasItem; -import static org.hamcrest.core.IsEqual.equalTo; -import static org.junit.Assert.assertThat; -import static org.springframework.integration.kafka.util.MessageUtils.decodeKey; -import static org.springframework.integration.kafka.util.MessageUtils.decodePayload; - -import java.util.ArrayList; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import com.gs.collections.api.RichIterable; -import com.gs.collections.api.bag.MutableBag; -import com.gs.collections.api.block.function.Function; -import com.gs.collections.api.block.function.Function2; -import com.gs.collections.api.block.procedure.Procedure; -import com.gs.collections.api.list.MutableList; -import com.gs.collections.api.map.MutableMap; -import com.gs.collections.api.multimap.list.MutableListMultimap; -import com.gs.collections.api.set.MutableSet; -import com.gs.collections.api.tuple.Pair; -import com.gs.collections.impl.factory.Sets; -import com.gs.collections.impl.multimap.list.SynchronizedPutFastListMultimap; -import com.gs.collections.impl.tuple.Tuples; -import kafka.serializer.StringDecoder; -import kafka.utils.VerifiableProperties; -import org.hamcrest.Matchers; - -import org.springframework.integration.kafka.core.ConnectionFactory; -import org.springframework.integration.kafka.core.Partition; -import org.springframework.integration.kafka.core.KafkaMessage; - -/** - * @author Marius Bogoevici - */ -public abstract class AbstractMessageListenerContainerTests extends AbstractBrokerTests { - - public void runMessageListenerTest(int maxReceiveSize, int concurrency, int partitionCount, int testMessageCount, - int divisionFactor, String compressionCodec, String topic) throws Exception { - - ConnectionFactory connectionFactory = getKafkaBrokerConnectionFactory(); - ArrayList readPartitions = new ArrayList(); - for (int i = 0; i < partitionCount; i++) { - if(i % divisionFactor == 0) { - readPartitions.add(new Partition(topic, i)); - } - } - final KafkaMessageListenerContainer kafkaMessageListenerContainer = - new KafkaMessageListenerContainer(connectionFactory, readPartitions.toArray(new Partition[readPartitions.size()])); - kafkaMessageListenerContainer.setMaxFetch(maxReceiveSize); - kafkaMessageListenerContainer.setConcurrency(concurrency); - - int expectedMessageCount = testMessageCount / divisionFactor; - - final MutableListMultimap receivedData = new SynchronizedPutFastListMultimap(); - final CountDownLatch latch = new CountDownLatch(expectedMessageCount); - kafkaMessageListenerContainer.setMessageListener(new MessageListener() { - @Override - public void onMessage(KafkaMessage message) { - StringDecoder decoder = new StringDecoder(new VerifiableProperties()); - receivedData.put(message.getMetadata().getPartition().getId(), - new KeyedMessageWithOffset(decodeKey(message, decoder), - decodePayload(message, decoder), message.getMetadata().getOffset(), - Thread.currentThread().getName(), - message.getMetadata().getPartition().getId())); - latch.countDown(); - } - }); - - kafkaMessageListenerContainer.start(); - - createMessageSender(compressionCodec).send(createMessages(testMessageCount, topic, partitionCount)); - - latch.await((expectedMessageCount/5000) + 1, TimeUnit.MINUTES); - kafkaMessageListenerContainer.stop(); - - assertThat(receivedData.valuesView().toList(), hasSize(expectedMessageCount)); - assertThat(latch.getCount(), equalTo(0L)); - System.out.println("All messages received ... checking "); - - validateMessageReceipt(receivedData, concurrency, partitionCount, expectedMessageCount, - divisionFactor); - - } - - @SuppressWarnings("serial") - public void validateMessageReceipt(MutableListMultimap receivedData, - int concurrency, int partitionCount, int expectedMessageCount, int divisionFactor) { - // Group messages received by processing thread - MutableListMultimap messagesByThread = receivedData.valuesView().toList().groupBy(new Function() { - @Override - public String valueOf(KeyedMessageWithOffset object) { - return object.getThreadName(); - } - }); - - // Execution has taken place on as many distinct threads as configured - messagesByThread.keysView().forEach(new Procedure() { - @Override - public void value(String s) { - System.out.println(s); - } - }); - assertThat(messagesByThread.keysView().size(), Matchers.equalTo(concurrency)); - - // Group partitions by thread - MutableMap> partitionsByThread = messagesByThread.toMap().collect(new Function2, Pair>>() { - @Override - public Pair> value(String argument1, RichIterable argument2) { - return Tuples.pair(argument1, argument2.collect(new Function() { - @Override - public Integer valueOf(KeyedMessageWithOffset object) { - return object.getPartition(); - } - }).toSet()); - } - }); - - // Messages from a partition have been executed on the same thread and groups are mutually exclusive - final MutableSet validatedPartitions = Sets.mutable.of(); - partitionsByThread.valuesView().forEach(new Procedure>() { - @Override - public void value(MutableSet partitions) { - assertThat(validatedPartitions.intersect(partitions), empty()); - validatedPartitions.addAll(partitions); - } - }); - - // All partitions are accounted for, but only the ones that we were expecting to read from - for (int i = 0; i < partitionCount; i++) { - if (i % divisionFactor == 0) { - assertThat(validatedPartitions, hasItem(i)); - } else { - assertThat(validatedPartitions, not(hasItem(i))); - } - } - - // Sort data by payload in order to identify duplicates - MutableList sortedPayloads = receivedData.valuesView().toList().collect(new Function() { - @Override - public String valueOf(KeyedMessageWithOffset object) { - return object.getPayload(); - } - }).sortThis(); - - // Remove unique values - what is left are duplicates - MutableBag duplicates = sortedPayloads.toBag(); - duplicates.removeAll(sortedPayloads.toSet()); - - // The final set has exactly the same size as the message count - assertThat(sortedPayloads, hasSize(expectedMessageCount)); - - // There are no duplicates - all messages have been received only once - assertThat(duplicates, hasSize(0)); - - // Group offsets by partition - MutableMap> offsetsByPartition = receivedData.toMap().collect(new Function2, Pair>>() { - @Override - public Pair> value(Integer partition, RichIterable argument2) { - return Tuples.pair(partition, argument2.collect(new Function() { - @Override - public Long valueOf(KeyedMessageWithOffset object) { - return object.getOffset(); - } - }).toList()); - } - }); - - // Check that the sequence of offsets has been processed in ascending order, with no gaps - for (MutableList offsetsForPartition : offsetsByPartition.valuesView()) { - for (int i = 0; i < offsetsForPartition.size() - 1; i++) { - assertThat(offsetsForPartition.get(i+1), equalTo(offsetsForPartition.get(i) + 1)); - } - } - } - - static class KeyedMessageWithOffset { - - String key; - - String payload; - - Long offset; - - String threadName; - - int partition; - - public KeyedMessageWithOffset(String key, String payload, Long offset, String threadName, int partition) { - this.key = key; - this.payload = payload; - this.offset = offset; - this.threadName = threadName; - this.partition = partition; - } - - public String getKey() { - return key; - } - - public String getPayload() { - return payload; - } - - public Long getOffset() { - return offset; - } - - public String getThreadName() { - return threadName; - } - - public int getPartition() { - return partition; - } - } -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/ChannelAdapterShutdownTests-context.xml b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/ChannelAdapterShutdownTests-context.xml deleted file mode 100644 index bea3916c6a..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/ChannelAdapterShutdownTests-context.xml +++ /dev/null @@ -1,47 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/ChannelAdapterShutdownTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/ChannelAdapterShutdownTests.java deleted file mode 100644 index 6d578b619f..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/ChannelAdapterShutdownTests.java +++ /dev/null @@ -1,92 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package org.springframework.integration.kafka.listener; - -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; - -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import org.junit.Rule; -import org.junit.Test; -import org.springframework.context.support.ClassPathXmlApplicationContext; -import org.springframework.integration.annotation.ServiceActivator; -import org.springframework.integration.kafka.rule.KafkaEmbedded; -import org.springframework.integration.kafka.rule.KafkaRule; -import org.springframework.messaging.Message; -import org.springframework.util.StopWatch; - -/** - * @author Marius Bogoevici - */ -public class ChannelAdapterShutdownTests extends AbstractMessageListenerContainerTests { - - @Rule - public final KafkaRule kafkaEmbeddedBrokerRule = new KafkaEmbedded(1); - - @Override - public KafkaRule getKafkaRule() { - return this.kafkaEmbeddedBrokerRule; - } - - @Test - public void testShutdownNotBlocking() throws Exception { - - System.setProperty("kafka.test.topic", TEST_TOPIC); - - int partitionCount = 1; - - createTopic(TEST_TOPIC, partitionCount, 1, 1); - - createMessageSender("none").send(createMessages(100, TEST_TOPIC, partitionCount)); - - ClassPathXmlApplicationContext context = - new ClassPathXmlApplicationContext("ChannelAdapterShutdownTests-context.xml", - ChannelAdapterShutdownTests.class); - TripLatchService tripLatchService = context.getBean(TripLatchService.class); - assertTrue("Message reception hasn't started", tripLatchService.getLatch().await(10, TimeUnit.SECONDS)); - StopWatch stopWatch = new StopWatch(); - stopWatch.start(); - context.stop(); - stopWatch.stop(); - long shutDownTime = stopWatch.getTotalTimeMillis(); - if (shutDownTime > 25000) { - fail("Context stopping only on timeout"); - } - } - - /** - * A simple gateway latch holder that trips when a message passes through. Used to detect that the message listener - * container has started receiving messages - */ - public static class TripLatchService { - - private final CountDownLatch latch = new CountDownLatch(1); - - public CountDownLatch getLatch() { - return latch; - } - - @ServiceActivator - public Message trip(Message message) { - latch.countDown(); - return message; - } - } -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/ChannelAdapterWithXmlConfigurationTests-context.xml b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/ChannelAdapterWithXmlConfigurationTests-context.xml deleted file mode 100644 index aaa9801d32..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/ChannelAdapterWithXmlConfigurationTests-context.xml +++ /dev/null @@ -1,39 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/ChannelAdapterWithXmlConfigurationTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/ChannelAdapterWithXmlConfigurationTests.java deleted file mode 100644 index 3273ebcdc4..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/ChannelAdapterWithXmlConfigurationTests.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package org.springframework.integration.kafka.listener; - -import static org.hamcrest.Matchers.notNullValue; - - -import org.junit.Assert; -import org.junit.Rule; -import org.junit.Test; - -import org.springframework.context.support.ClassPathXmlApplicationContext; -import org.springframework.integration.channel.QueueChannel; -import org.springframework.integration.kafka.rule.KafkaEmbedded; -import org.springframework.integration.kafka.rule.KafkaRule; -import org.springframework.messaging.Message; - -/** - * @author Marius Bogoevici - */ -public class ChannelAdapterWithXmlConfigurationTests extends AbstractMessageListenerContainerTests { - - @Rule - public final KafkaRule kafkaEmbeddedBrokerRule = new KafkaEmbedded(1); - - @Override - public KafkaRule getKafkaRule() { - return this.kafkaEmbeddedBrokerRule; - } - - @Test - public void testConsumptionWithXmlConfiguration() throws Exception { - - System.setProperty("kafka.test.topic", TEST_TOPIC); - - int partitionCount = 1; - - createTopic(TEST_TOPIC, partitionCount, 1, 1); - - createMessageSender("none").send(createMessages(100, TEST_TOPIC, partitionCount)); - - ClassPathXmlApplicationContext context = - new ClassPathXmlApplicationContext("ChannelAdapterWithXmlConfigurationTests-context.xml", - ChannelAdapterWithXmlConfigurationTests.class); - - QueueChannel output = context.getBean("output", QueueChannel.class); - - for (int i = 0; i < 100; i++) { - Message received = output.receive(1000); - Assert.assertThat(received, notNullValue()); - } - } -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/ChannelSendingMessageListener.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/ChannelSendingMessageListener.java deleted file mode 100644 index 9eda772104..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/ChannelSendingMessageListener.java +++ /dev/null @@ -1,61 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package org.springframework.integration.kafka.listener; - -import org.springframework.beans.BeansException; -import org.springframework.context.ApplicationContext; -import org.springframework.context.ApplicationContextAware; -import org.springframework.integration.kafka.core.KafkaMessage; -import org.springframework.integration.kafka.listener.MessageListener; -import org.springframework.integration.support.MessageBuilder; -import org.springframework.integration.support.channel.BeanFactoryChannelResolver; -import org.springframework.messaging.MessageChannel; -import org.springframework.messaging.core.DestinationResolver; - -/** - * @author Marius Bogoevici - */ -public class ChannelSendingMessageListener implements MessageListener, ApplicationContextAware { - - private String channelName; - - private MessageChannel messageChannel; - - public String getChannelName() { - return channelName; - } - - public void setChannelName(String channelName) { - this.channelName = channelName; - } - - DestinationResolver destinationResolver; - - @Override - public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { - this.destinationResolver = new BeanFactoryChannelResolver(applicationContext); - messageChannel = destinationResolver.resolveDestination(channelName); - } - - @Override - public void onMessage(KafkaMessage message) { - byte b[] = new byte[message.getMessage().payloadSize()]; - message.getMessage().payload().get(b); - messageChannel.send(MessageBuilder.withPayload(new String(b)).build()); - } -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/DefaultConnectionFactoryTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/DefaultConnectionFactoryTests.java deleted file mode 100644 index 202fa92a4d..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/DefaultConnectionFactoryTests.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package org.springframework.integration.kafka.listener; - -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.Matchers.instanceOf; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.fail; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; - -import java.util.Collections; - -import org.junit.Rule; -import org.junit.Test; - -import org.springframework.integration.kafka.core.BrokerAddress; -import org.springframework.integration.kafka.core.BrokerAddressListConfiguration; -import org.springframework.integration.kafka.core.Configuration; -import org.springframework.integration.kafka.core.Connection; -import org.springframework.integration.kafka.core.DefaultConnectionFactory; -import org.springframework.integration.kafka.core.ZookeeperConfiguration; -import org.springframework.integration.kafka.rule.KafkaEmbedded; -import org.springframework.integration.kafka.support.ZookeeperConnect; - -/** - * @author Marius Bogoevici - * @author Artem Bilan - */ -public class DefaultConnectionFactoryTests extends AbstractBrokerTests { - - @Rule - public KafkaEmbedded kafkaEmbeddedBrokerRule = new KafkaEmbedded(1); - - @Override - public KafkaEmbedded getKafkaRule() { - return kafkaEmbeddedBrokerRule; - } - - @Test - public void testCreateConnectionFactoryWithBrokerList() throws Exception { - - createTopic(TEST_TOPIC, 1, 1, 1); - - BrokerAddress[] brokerAddresses = getKafkaRule().getBrokerAddresses(); - DefaultConnectionFactory connectionFactory = - new DefaultConnectionFactory(new BrokerAddressListConfiguration(brokerAddresses)); - connectionFactory.afterPropertiesSet(); - Connection connection = connectionFactory.connect(brokerAddresses[0]); - assertNotNull(connection); - } - - @Test - public void testCreateConnectionFactoryWithZookeeper() throws Exception { - - createTopic(TEST_TOPIC, 1, 1, 1); - - ZookeeperConnect zookeeperConnect = new ZookeeperConnect(); - zookeeperConnect.setZkConnect(kafkaEmbeddedBrokerRule.getZookeeperConnectionString()); - DefaultConnectionFactory connectionFactory = - new DefaultConnectionFactory(new ZookeeperConfiguration(zookeeperConnect)); - connectionFactory.afterPropertiesSet(); - Connection connection = connectionFactory.connect(getKafkaRule().getBrokerAddresses()[0]); - assertNotNull(connection); - } - - @Test - public void testNpeFromClientUtils() throws Exception { - createTopic(TEST_TOPIC, 1, 1, 1); - Configuration configuration = mock(Configuration.class); - when(configuration.getBrokerAddresses()) - .thenReturn(Collections.singletonList(BrokerAddress.fromAddress("localhost,:9092"))); - DefaultConnectionFactory connectionFactory = new DefaultConnectionFactory(configuration); - connectionFactory.afterPropertiesSet(); - try { - connectionFactory.refreshMetadata(Collections.singletonList(TEST_TOPIC)); - fail("IllegalStateException expected"); - } - catch (Exception e) { - assertThat(e, instanceOf(IllegalStateException.class)); - assertThat(e.getMessage(), containsString("Can not parse Kafka Brokers for")); - assertThat(e.getCause(), instanceOf(NullPointerException.class)); - } - } - -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/DefaultConnectionTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/DefaultConnectionTests.java deleted file mode 100644 index 92006aa2e8..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/DefaultConnectionTests.java +++ /dev/null @@ -1,133 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package org.springframework.integration.kafka.listener; - -import org.apache.kafka.clients.producer.Producer; -import org.junit.Assert; -import org.junit.Ignore; -import org.junit.Rule; -import org.junit.Test; - -import org.springframework.integration.kafka.core.Connection; -import org.springframework.integration.kafka.core.DefaultConnection; -import org.springframework.integration.kafka.core.FetchRequest; -import org.springframework.integration.kafka.core.KafkaMessage; -import org.springframework.integration.kafka.core.KafkaMessageBatch; -import org.springframework.integration.kafka.core.Partition; -import org.springframework.integration.kafka.core.Result; -import org.springframework.integration.kafka.rule.KafkaEmbedded; -import org.springframework.integration.kafka.serializer.common.StringDecoder; -import org.springframework.integration.kafka.util.MessageUtils; - -/** - * @author Marius Bogoevici - */ -public class DefaultConnectionTests extends AbstractBrokerTests { - - @Rule - public KafkaEmbedded kafkaEmbeddedBrokerRule = new KafkaEmbedded(1); - - @Override - public KafkaEmbedded getKafkaRule() { - return kafkaEmbeddedBrokerRule; - } - - @Test - public void testFetchPartitionMetadata() throws Exception { - createTopic(TEST_TOPIC, 1, 1, 1); - Connection brokerConnection = - new DefaultConnection(getKafkaRule().getBrokerAddresses()[0], "client", 64*1024, 3000, 1, 10000); - Partition partition = new Partition(TEST_TOPIC, 0); - Result result = brokerConnection.fetchInitialOffset(-1, partition); - Assert.assertEquals(0, result.getErrors().size()); - Assert.assertEquals(1, result.getResults().size()); - Assert.assertEquals(Long.valueOf(0), result.getResults().get(partition)); - } - - @Test - public void testReceiveMessages() throws Exception { - createTopic(TEST_TOPIC, 1, 1, 1); - createMessageSender("none").send(createMessages(10, TEST_TOPIC, 1)); - Connection brokerConnection = - new DefaultConnection(getKafkaRule().getBrokerAddresses()[0], "client", 64*1024, 3000, 1, 10000); - Partition partition = new Partition(TEST_TOPIC, 0); - FetchRequest fetchRequest = new FetchRequest(partition, 0L, 1000); - Result result = brokerConnection.fetch(fetchRequest); - Assert.assertEquals(0, result.getErrors().size()); - Assert.assertEquals(1, result.getResults().size()); - Assert.assertEquals(10, result.getResults().get(partition).getMessages().size()); - Assert.assertEquals(10,result.getResults().get(partition).getHighWatermark()); - StringDecoder decoder = new StringDecoder(); - int i = 0; - for (KafkaMessage kafkaMessage : result.getResults().get(partition).getMessages()) { - Assert.assertEquals("Key " + i, MessageUtils.decodeKey(kafkaMessage, decoder)); - Assert.assertEquals("Message " + i, MessageUtils.decodePayload(kafkaMessage, decoder)); - i++; - } - } - - @Test - public void testReceiveMessagesWithGZipCompression() throws Exception { - createTopic(TEST_TOPIC, 1, 1, 1); - createMessageSender("gzip").send(createMessages(10, TEST_TOPIC, 1)); - Connection brokerConnection = - new DefaultConnection(getKafkaRule().getBrokerAddresses()[0], "client", 64*1024, 3000, 1, 10000); - Partition partition = new Partition(TEST_TOPIC, 0); - FetchRequest fetchRequest = new FetchRequest(partition, 0L, 1000); - Result result = brokerConnection.fetch(fetchRequest); - Assert.assertEquals(0, result.getErrors().size()); - Assert.assertEquals(1, result.getResults().size()); - Assert.assertEquals(10, result.getResults().get(partition).getMessages().size()); - Assert.assertEquals(10,result.getResults().get(partition).getHighWatermark()); - StringDecoder decoder = new StringDecoder(); - int i = 0; - for (KafkaMessage kafkaMessage : result.getResults().get(partition).getMessages()) { - Assert.assertEquals("Key " + i, MessageUtils.decodeKey(kafkaMessage, decoder)); - Assert.assertEquals("Message " + i, MessageUtils.decodePayload(kafkaMessage, decoder)); - i++; - } - } - - @Test - @Ignore - /** - * Since the test relies on the native library we can't test it on all environments, - * because we may not have permission to load dll(so). - */ - public void testReceiveMessagesWithSnappyCompression() throws Exception { - createTopic(TEST_TOPIC, 1, 1, 1); - createMessageSender("snappy").send( createMessages(10, TEST_TOPIC,1)); - Connection brokerConnection = - new DefaultConnection(getKafkaRule().getBrokerAddresses()[0], "client", 64*1024, 3000, 1, 10000); - Partition partition = new Partition(TEST_TOPIC, 0); - FetchRequest fetchRequest = new FetchRequest(partition, 0L, 1000); - Result result = brokerConnection.fetch(fetchRequest); - Assert.assertEquals(0, result.getErrors().size()); - Assert.assertEquals(1, result.getResults().size()); - Assert.assertEquals(10, result.getResults().get(partition).getMessages().size()); - Assert.assertEquals(10,result.getResults().get(partition).getHighWatermark()); - StringDecoder decoder = new StringDecoder(); - int i = 0; - for (KafkaMessage kafkaMessage : result.getResults().get(partition).getMessages()) { - Assert.assertEquals("Key " + i, MessageUtils.decodeKey(kafkaMessage, decoder)); - Assert.assertEquals("Message " + i, MessageUtils.decodePayload(kafkaMessage, decoder)); - i++; - } - } - -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/KafkaMessageDrivenChannelAdapterTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/KafkaMessageDrivenChannelAdapterTests.java deleted file mode 100644 index 935f38537e..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/KafkaMessageDrivenChannelAdapterTests.java +++ /dev/null @@ -1,246 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package org.springframework.integration.kafka.listener; - -import static org.hamcrest.CoreMatchers.nullValue; -import static org.hamcrest.Matchers.containsString; -import static org.hamcrest.collection.IsCollectionWithSize.hasSize; -import static org.hamcrest.core.IsEqual.equalTo; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.fail; -import static org.mockito.Mockito.mock; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import org.junit.Rule; -import org.junit.Test; - -import org.springframework.beans.factory.BeanFactory; -import org.springframework.integration.kafka.core.ConnectionFactory; -import org.springframework.integration.kafka.core.Partition; -import org.springframework.integration.kafka.inbound.KafkaMessageDrivenChannelAdapter; -import org.springframework.integration.kafka.rule.KafkaEmbedded; -import org.springframework.integration.kafka.rule.KafkaRule; -import org.springframework.integration.kafka.serializer.common.StringDecoder; -import org.springframework.integration.kafka.support.KafkaHeaders; -import org.springframework.integration.metadata.SimpleMetadataStore; -import org.springframework.messaging.Message; -import org.springframework.messaging.MessageChannel; - -import com.gs.collections.api.multimap.list.MutableListMultimap; -import com.gs.collections.impl.list.mutable.FastList; -import com.gs.collections.impl.multimap.list.SynchronizedPutFastListMultimap; - -/** - * @author Marius Bogoevici - */ -public class KafkaMessageDrivenChannelAdapterTests extends AbstractMessageListenerContainerTests { - - @Rule - public final KafkaEmbedded kafkaEmbeddedBrokerRule = new KafkaEmbedded(1); - - @Override - public KafkaRule getKafkaRule() { - return kafkaEmbeddedBrokerRule; - } - - @Test - @SuppressWarnings("serial") - public void testLowVolumeLowConcurrency() throws Exception { - int partitionCount = 5; - - createTopic(TEST_TOPIC, partitionCount, 1, 1); - - ConnectionFactory connectionFactory = getKafkaBrokerConnectionFactory(); - ArrayList readPartitions = new ArrayList(); - for (int i = 0; i < partitionCount; i++) { - readPartitions.add(new Partition(TEST_TOPIC, i)); - } - - final KafkaMessageListenerContainer kafkaMessageListenerContainer = - new KafkaMessageListenerContainer(connectionFactory, - readPartitions.toArray(new Partition[readPartitions.size()])); - kafkaMessageListenerContainer.setMaxFetch(100); - kafkaMessageListenerContainer.setConcurrency(2); - - try { - kafkaMessageListenerContainer.setQueueSize(1000); - fail("expected exception"); - } - catch (IllegalArgumentException e) { - assertThat(e.getMessage(), containsString("power of 2")); - } - - try { - kafkaMessageListenerContainer.setQueueSize(Integer.MIN_VALUE); - fail("expected exception"); - } - catch (IllegalArgumentException e) { - assertThat(e.getMessage(), containsString("positive number")); - } - - kafkaMessageListenerContainer.setQueueSize(1024); - - int expectedMessageCount = 100; - - final MutableListMultimap receivedData = - new SynchronizedPutFastListMultimap(); - final CountDownLatch latch = new CountDownLatch(expectedMessageCount); - - KafkaMessageDrivenChannelAdapter kafkaMessageDrivenChannelAdapter = - new KafkaMessageDrivenChannelAdapter(kafkaMessageListenerContainer); - - StringDecoder decoder = new StringDecoder(); - kafkaMessageDrivenChannelAdapter.setKeyDecoder(decoder); - kafkaMessageDrivenChannelAdapter.setPayloadDecoder(decoder); - kafkaMessageDrivenChannelAdapter.setBeanFactory(mock(BeanFactory.class)); - kafkaMessageDrivenChannelAdapter.setOutputChannel(new MessageChannel() { - @Override - public boolean send(Message message) { - boolean addedSuccessfully = receivedData.put( - (Integer) message.getHeaders().get(KafkaHeaders.PARTITION_ID), - new KeyedMessageWithOffset( - (String) message.getHeaders().get(KafkaHeaders.MESSAGE_KEY), - (String) message.getPayload(), - (Long) message.getHeaders().get(KafkaHeaders.OFFSET), - Thread.currentThread().getName(), - (Integer) message.getHeaders().get(KafkaHeaders.PARTITION_ID))); - latch.countDown(); - return addedSuccessfully; - } - - - @Override - public boolean send(Message message, long timeout) { - return send(message); - } - }); - - kafkaMessageDrivenChannelAdapter.afterPropertiesSet(); - kafkaMessageDrivenChannelAdapter.start(); - - createMessageSender("none").send(createMessages(100, TEST_TOPIC, partitionCount)); - - latch.await((expectedMessageCount / 5000) + 1, TimeUnit.MINUTES); - kafkaMessageListenerContainer.stop(); - - assertThat(receivedData.valuesView().toList(), hasSize(expectedMessageCount)); - assertThat(latch.getCount(), equalTo(0L)); - System.out.println("All messages received ... checking "); - - validateMessageReceipt(receivedData, 2, partitionCount, expectedMessageCount, 1); - - } - - @Test - @SuppressWarnings("serial") - public void testManualAck() throws Exception { - int partitionCount = 5; - - createTopic(TEST_TOPIC, partitionCount, 1, 1); - - ConnectionFactory connectionFactory = getKafkaBrokerConnectionFactory(); - ArrayList readPartitions = new ArrayList(); - for (int i = 0; i < partitionCount; i++) { - readPartitions.add(new Partition(TEST_TOPIC, i)); - } - - final KafkaMessageListenerContainer kafkaMessageListenerContainer = - new KafkaMessageListenerContainer(connectionFactory, - readPartitions.toArray(new Partition[readPartitions.size()])); - MetadataStoreOffsetManager offsetManager = new MetadataStoreOffsetManager(connectionFactory); - SimpleMetadataStore metadataStore = new SimpleMetadataStore(); - offsetManager.setMetadataStore(metadataStore); - kafkaMessageListenerContainer.setOffsetManager(offsetManager); - kafkaMessageListenerContainer.setMaxFetch(100); - kafkaMessageListenerContainer.setConcurrency(2); - - int expectedMessageCount = 100; - - final MutableListMultimap receivedData = - new SynchronizedPutFastListMultimap(); - final List acknowledgments = Collections.synchronizedList(new ArrayList()); - final CountDownLatch latch = new CountDownLatch(expectedMessageCount); - - KafkaMessageDrivenChannelAdapter kafkaMessageDrivenChannelAdapter = - new KafkaMessageDrivenChannelAdapter(kafkaMessageListenerContainer); - - StringDecoder decoder = new StringDecoder(); - kafkaMessageDrivenChannelAdapter.setKeyDecoder(decoder); - kafkaMessageDrivenChannelAdapter.setPayloadDecoder(decoder); - kafkaMessageDrivenChannelAdapter.setBeanFactory(mock(BeanFactory.class)); - kafkaMessageDrivenChannelAdapter.setOutputChannel(new MessageChannel() { - @Override - public boolean send(Message message) { - boolean addedSuccessfully = receivedData.put( - (Integer) message.getHeaders().get(KafkaHeaders.PARTITION_ID), - new KeyedMessageWithOffset( - (String) message.getHeaders().get(KafkaHeaders.MESSAGE_KEY), - (String) message.getPayload(), - (Long) message.getHeaders().get(KafkaHeaders.OFFSET), - Thread.currentThread().getName(), - (Integer) message.getHeaders().get(KafkaHeaders.PARTITION_ID))); - acknowledgments.add(message.getHeaders().get(KafkaHeaders.ACKNOWLEDGMENT, Acknowledgment.class)); - latch.countDown(); - return addedSuccessfully; - } - - - @Override - public boolean send(Message message, long timeout) { - return send(message); - } - }); - - kafkaMessageDrivenChannelAdapter.setAutoCommitOffset(false); - kafkaMessageDrivenChannelAdapter.afterPropertiesSet(); - kafkaMessageDrivenChannelAdapter.start(); - - createMessageSender("none").send(createMessages(100, TEST_TOPIC, partitionCount)); - - latch.await((expectedMessageCount / 5000) + 1, TimeUnit.MINUTES); - kafkaMessageListenerContainer.stop(); - - assertThat(receivedData.valuesView().toList(), hasSize(expectedMessageCount)); - assertThat(latch.getCount(), equalTo(0L)); - System.out.println("All messages received ... checking "); - - validateMessageReceipt(receivedData, 2, partitionCount, expectedMessageCount, 1); - - // at this point, all messages have been processed but not acknowledged - for (Partition readPartition : readPartitions) { - assertThat(metadataStore.get(offsetManager.generateKey(readPartition)), nullValue()); - } - - // Now, we did acknowledge them in the reverse order. - // This way we check that only the highest value was acknowledged - for (Acknowledgment acknowledgment : FastList.newList(acknowledgments).reverseThis()) { - acknowledgment.acknowledge(); - } - - // now they are all acknowledged - for (Partition readPartition : readPartitions) { - assertThat(metadataStore.get(offsetManager.generateKey(readPartition)), equalTo(String.valueOf(20))); - } - } - -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/KafkaMessageDrivenChannelAdapterWithKafkaOffsetManagerTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/KafkaMessageDrivenChannelAdapterWithKafkaOffsetManagerTests.java deleted file mode 100644 index 45b7d21e97..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/KafkaMessageDrivenChannelAdapterWithKafkaOffsetManagerTests.java +++ /dev/null @@ -1,139 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.listener; - -import static org.hamcrest.collection.IsCollectionWithSize.hasSize; -import static org.hamcrest.core.IsEqual.equalTo; -import static org.junit.Assert.assertThat; -import static org.mockito.Mockito.mock; - -import java.util.ArrayList; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import org.junit.Rule; -import org.junit.Test; - -import org.springframework.beans.factory.BeanFactory; -import org.springframework.integration.channel.FixedSubscriberChannel; -import org.springframework.integration.kafka.core.ConnectionFactory; -import org.springframework.integration.kafka.core.Partition; -import org.springframework.integration.kafka.inbound.KafkaMessageDrivenChannelAdapter; -import org.springframework.integration.kafka.rule.KafkaEmbedded; -import org.springframework.integration.kafka.rule.KafkaRule; -import org.springframework.integration.kafka.serializer.common.StringDecoder; -import org.springframework.integration.kafka.support.KafkaHeaders; -import org.springframework.integration.kafka.support.ZookeeperConnect; -import org.springframework.messaging.Message; -import org.springframework.messaging.MessageHandler; -import org.springframework.messaging.MessagingException; - -import com.gs.collections.api.multimap.list.MutableListMultimap; -import com.gs.collections.impl.multimap.list.SynchronizedPutFastListMultimap; - -/** - * @author Marius Bogoevici - */ -public class KafkaMessageDrivenChannelAdapterWithKafkaOffsetManagerTests extends AbstractMessageListenerContainerTests { - - @Rule - public final KafkaEmbedded kafkaEmbeddedBrokerRule = new KafkaEmbedded(1); - - @Override - public KafkaRule getKafkaRule() { - return kafkaEmbeddedBrokerRule; - } - - @Test - public void testKafkaOffsetManager() throws Exception { - createTopic(TEST_TOPIC, 5, 1, 1); - - ConnectionFactory connectionFactory = getKafkaBrokerConnectionFactory(); - ArrayList readPartitions = new ArrayList(); - for (int i = 0; i < 5; i++) { - readPartitions.add(new Partition(TEST_TOPIC, i)); - } - - final KafkaMessageListenerContainer kafkaMessageListenerContainer = - new KafkaMessageListenerContainer(connectionFactory, - readPartitions.toArray(new Partition[readPartitions.size()])); - KafkaTopicOffsetManager offsetManager = new KafkaTopicOffsetManager( - new ZookeeperConnect(getKafkaRule().getZookeeperConnectionString()), "si-offsets"); - offsetManager.afterPropertiesSet(); - kafkaMessageListenerContainer.setOffsetManager(offsetManager); - kafkaMessageListenerContainer.setMaxFetch(100); - kafkaMessageListenerContainer.setConcurrency(2); - - int expectedMessageCount = 100; - - final MutableListMultimap receivedData = - new SynchronizedPutFastListMultimap(); - final CountDownLatch latch = new CountDownLatch(expectedMessageCount); - - KafkaMessageDrivenChannelAdapter kafkaMessageDrivenChannelAdapter = - new KafkaMessageDrivenChannelAdapter(kafkaMessageListenerContainer); - - StringDecoder decoder = new StringDecoder(); - kafkaMessageDrivenChannelAdapter.setKeyDecoder(decoder); - kafkaMessageDrivenChannelAdapter.setPayloadDecoder(decoder); - kafkaMessageDrivenChannelAdapter.setBeanFactory(mock(BeanFactory.class)); - kafkaMessageDrivenChannelAdapter.setOutputChannel(new FixedSubscriberChannel(new MessageHandler() { - - @Override - public void handleMessage(Message message) throws MessagingException { - receivedData.put( - (Integer) message.getHeaders().get(KafkaHeaders.PARTITION_ID), - new KeyedMessageWithOffset( - (String) message.getHeaders().get(KafkaHeaders.MESSAGE_KEY), - (String) message.getPayload(), - (Long) message.getHeaders().get(KafkaHeaders.OFFSET), - Thread.currentThread().getName(), - (Integer) message.getHeaders().get(KafkaHeaders.PARTITION_ID))); - latch.countDown(); - } - - })); - - kafkaMessageDrivenChannelAdapter.afterPropertiesSet(); - kafkaMessageDrivenChannelAdapter.start(); - - createMessageSender("none").send(createMessages(expectedMessageCount, TEST_TOPIC, 5)); - - latch.await((expectedMessageCount / 5000) + 1, TimeUnit.MINUTES); - kafkaMessageListenerContainer.stop(); - - assertThat(receivedData.valuesView().toList(), hasSize(expectedMessageCount)); - assertThat(latch.getCount(), equalTo(0L)); - System.out.println("All messages received ... checking "); - - validateMessageReceipt(receivedData, 2, 5, expectedMessageCount, 1); - - offsetManager.close(); - - KafkaTopicOffsetManager offsetManager2 = - new KafkaTopicOffsetManager(new ZookeeperConnect(getKafkaRule().getZookeeperConnectionString()), "si-offsets"); - offsetManager2.afterPropertiesSet(); - - assertThat(offsetManager2.getOffset(new Partition(TEST_TOPIC,0)),equalTo(20L)); - assertThat(offsetManager2.getOffset(new Partition(TEST_TOPIC,1)),equalTo(20L)); - assertThat(offsetManager2.getOffset(new Partition(TEST_TOPIC,2)),equalTo(20L)); - assertThat(offsetManager2.getOffset(new Partition(TEST_TOPIC,3)),equalTo(20L)); - assertThat(offsetManager2.getOffset(new Partition(TEST_TOPIC,4)),equalTo(20L)); - - } - -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/KafkaMessageDrivenChannelAdapterWithSpecialOffsetTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/KafkaMessageDrivenChannelAdapterWithSpecialOffsetTests.java deleted file mode 100644 index a2916be74c..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/KafkaMessageDrivenChannelAdapterWithSpecialOffsetTests.java +++ /dev/null @@ -1,172 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package org.springframework.integration.kafka.listener; - -import static org.hamcrest.collection.IsCollectionWithSize.hasSize; -import static org.hamcrest.core.IsEqual.equalTo; -import static org.junit.Assert.assertThat; -import static org.mockito.Mockito.mock; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import com.gs.collections.api.RichIterable; -import com.gs.collections.api.block.function.Function; -import com.gs.collections.api.multimap.list.MutableListMultimap; -import com.gs.collections.api.tuple.Pair; -import com.gs.collections.impl.list.mutable.FastList; -import com.gs.collections.impl.multimap.list.SynchronizedPutFastListMultimap; -import com.gs.collections.impl.utility.Iterate; - -import org.junit.Rule; -import org.junit.Test; - -import org.springframework.beans.factory.BeanFactory; -import org.springframework.integration.kafka.core.ConnectionFactory; -import org.springframework.integration.kafka.core.Partition; -import org.springframework.integration.kafka.inbound.KafkaMessageDrivenChannelAdapter; -import org.springframework.integration.kafka.rule.KafkaEmbedded; -import org.springframework.integration.kafka.rule.KafkaRule; -import org.springframework.integration.kafka.serializer.common.StringDecoder; -import org.springframework.integration.kafka.support.KafkaHeaders; -import org.springframework.messaging.Message; -import org.springframework.messaging.MessageChannel; - -/** - * @author Marius Bogoevici - */ -public class KafkaMessageDrivenChannelAdapterWithSpecialOffsetTests extends AbstractMessageListenerContainerTests { - - @Rule - public final KafkaEmbedded kafkaEmbeddedBrokerRule = new KafkaEmbedded(1); - - @Override - public KafkaRule getKafkaRule() { - return kafkaEmbeddedBrokerRule; - } - - @Test - @SuppressWarnings("serial") - public void testLowVolumeLowConcurrency() throws Exception { - - // we will send 300 messages: first 200, then another 100 - // we will start reading from all partitions at offset 100 - int expectedMessageCount = 200; - int partitionCount = 5; - - createTopic(TEST_TOPIC, partitionCount, 1, 1); - - ConnectionFactory connectionFactory = getKafkaBrokerConnectionFactory(); - ArrayList readPartitions = new ArrayList(); - Map startingOffsets = new HashMap(); - for (int i = 0; i < partitionCount; i++) { - Partition partition = new Partition(TEST_TOPIC, i); - readPartitions.add(partition); - startingOffsets.put(partition, 20L); - } - - final KafkaMessageListenerContainer kafkaMessageListenerContainer = - new KafkaMessageListenerContainer(connectionFactory, - readPartitions.toArray(new Partition[readPartitions.size()])); - kafkaMessageListenerContainer.setMaxFetch(100); - kafkaMessageListenerContainer.setConcurrency(2); - MetadataStoreOffsetManager offsetManager = new MetadataStoreOffsetManager(connectionFactory, startingOffsets); - kafkaMessageListenerContainer.setOffsetManager(offsetManager); - - // we send 200 messages - createMessageSender("none").send(createMessagesInRange(0, 199, TEST_TOPIC, partitionCount)); - - final MutableListMultimap receivedData = - new SynchronizedPutFastListMultimap(); - final CountDownLatch latch = new CountDownLatch(expectedMessageCount); - - KafkaMessageDrivenChannelAdapter kafkaMessageDrivenChannelAdapter = - new KafkaMessageDrivenChannelAdapter(kafkaMessageListenerContainer); - - StringDecoder decoder = new StringDecoder(); - kafkaMessageDrivenChannelAdapter.setKeyDecoder(decoder); - kafkaMessageDrivenChannelAdapter.setPayloadDecoder(decoder); - kafkaMessageDrivenChannelAdapter.setBeanFactory(mock(BeanFactory.class)); - kafkaMessageDrivenChannelAdapter.setOutputChannel(new MessageChannel() { - @Override - public boolean send(Message message) { - boolean addedSuccessfully = receivedData.put( - (Integer) message.getHeaders().get(KafkaHeaders.PARTITION_ID), - new KeyedMessageWithOffset( - (String) message.getHeaders().get(KafkaHeaders.MESSAGE_KEY), - (String) message.getPayload(), - (Long) message.getHeaders().get(KafkaHeaders.OFFSET), - Thread.currentThread().getName(), - (Integer) message.getHeaders().get(KafkaHeaders.PARTITION_ID))); - latch.countDown(); - return addedSuccessfully; - } - - - @Override - public boolean send(Message message, long timeout) { - return send(message); - } - }); - - kafkaMessageDrivenChannelAdapter.afterPropertiesSet(); - kafkaMessageDrivenChannelAdapter.start(); - - createMessageSender("none").send(createMessagesInRange(200, 299, TEST_TOPIC, partitionCount)); - - latch.await((expectedMessageCount / 5000) + 1, TimeUnit.MINUTES); - kafkaMessageListenerContainer.stop(); - - assertThat(receivedData.valuesView().toList(), hasSize(expectedMessageCount)); - assertThat(latch.getCount(), equalTo(0L)); - System.out.println("All messages received ... checking "); - - validateMessageReceipt(receivedData, 2, partitionCount, expectedMessageCount, 1); - - // For all received messages - Collection allReceivedMessages = - Iterate.flatCollect(receivedData.keyMultiValuePairsView(), - new Function>, - RichIterable>() { - - @Override - public RichIterable valueOf(Pair> object) { - return object.getTwo(); - } - - }); - - // We extract the sequence value, i.e. "Message xx" - Integer minValueInMessage = FastList.newList(allReceivedMessages).collect(new Function() { - @Override - public Integer valueOf(KeyedMessageWithOffset object) { - return Integer.parseInt(object.getPayload().split(" ")[1]); - } - }).min(); - - // The lowest received value is 100. That is correct, because messages are evenly distributed across partitions - // and we start reading only at offset 20 - assertThat(minValueInMessage, equalTo(100)); - } - -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/KafkaMessageDrivenChannelAdapterWithWrongOffsetTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/KafkaMessageDrivenChannelAdapterWithWrongOffsetTests.java deleted file mode 100644 index b0aa553307..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/KafkaMessageDrivenChannelAdapterWithWrongOffsetTests.java +++ /dev/null @@ -1,327 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package org.springframework.integration.kafka.listener; - -import static org.hamcrest.collection.IsCollectionWithSize.hasSize; -import static org.hamcrest.core.IsEqual.equalTo; -import static org.junit.Assert.assertThat; -import static org.mockito.Mockito.mock; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import com.gs.collections.api.RichIterable; -import com.gs.collections.api.block.function.Function; -import com.gs.collections.api.multimap.list.MutableListMultimap; -import com.gs.collections.api.tuple.Pair; -import com.gs.collections.impl.list.mutable.FastList; -import com.gs.collections.impl.multimap.list.SynchronizedPutFastListMultimap; -import com.gs.collections.impl.utility.Iterate; - -import org.junit.Rule; -import org.junit.Test; - -import org.springframework.beans.factory.BeanFactory; -import org.springframework.integration.kafka.core.ConnectionFactory; -import org.springframework.integration.kafka.core.Partition; -import org.springframework.integration.kafka.inbound.KafkaMessageDrivenChannelAdapter; -import org.springframework.integration.kafka.rule.KafkaEmbedded; -import org.springframework.integration.kafka.rule.KafkaRule; -import org.springframework.integration.kafka.serializer.common.StringDecoder; -import org.springframework.integration.kafka.support.KafkaHeaders; -import org.springframework.messaging.Message; -import org.springframework.messaging.MessageChannel; -import org.springframework.util.StringUtils; - -/** - * @author Marius Bogoevici - */ -public class KafkaMessageDrivenChannelAdapterWithWrongOffsetTests extends AbstractMessageListenerContainerTests { - - @Rule - public final KafkaEmbedded kafkaEmbeddedBrokerRule = new KafkaEmbedded(1); - - @Override - public KafkaRule getKafkaRule() { - return kafkaEmbeddedBrokerRule; - } - - @Test - @SuppressWarnings("serial") - public void testLowVolumeLowConcurrencyResetAtEarliest() throws Exception { - - // we will send 300 messages: first 200, then another 100 - // we will start reading from all partitions at offset 100 - int expectedMessageCount = 300; - - int partitionCount = 5; - createTopic(TEST_TOPIC, partitionCount, 1, 1); - - ConnectionFactory connectionFactory = getKafkaBrokerConnectionFactory(); - ArrayList readPartitions = new ArrayList(); - Map startingOffsets = new HashMap(); - for (int i = 0; i < partitionCount; i++) { - Partition partition = new Partition(TEST_TOPIC, i); - readPartitions.add(partition); - startingOffsets.put(partition, 900L); - } - - final KafkaMessageListenerContainer kafkaMessageListenerContainer = - new KafkaMessageListenerContainer(connectionFactory, - readPartitions.toArray(new Partition[readPartitions.size()])); - kafkaMessageListenerContainer.setMaxFetch(100); - kafkaMessageListenerContainer.setConcurrency(2); - MetadataStoreOffsetManager offsetManager = new MetadataStoreOffsetManager(connectionFactory, startingOffsets); - kafkaMessageListenerContainer.setOffsetManager(offsetManager); - - // we send 200 messages - createMessageSender("none").send(createMessagesInRange(0, 199, TEST_TOPIC, partitionCount)); - - final MutableListMultimap receivedData = - new SynchronizedPutFastListMultimap(); - final CountDownLatch latch = new CountDownLatch(expectedMessageCount); - - KafkaMessageDrivenChannelAdapter kafkaMessageDrivenChannelAdapter = - new KafkaMessageDrivenChannelAdapter(kafkaMessageListenerContainer); - - StringDecoder decoder = new StringDecoder(); - kafkaMessageDrivenChannelAdapter.setKeyDecoder(decoder); - kafkaMessageDrivenChannelAdapter.setPayloadDecoder(decoder); - kafkaMessageDrivenChannelAdapter.setBeanFactory(mock(BeanFactory.class)); - - kafkaMessageDrivenChannelAdapter.setOutputChannel(new MessageChannel() { - @Override - public boolean send(Message message) { - boolean addedSuccessfully = receivedData.put( - (Integer) message.getHeaders().get(KafkaHeaders.PARTITION_ID), - new KeyedMessageWithOffset( - (String) message.getHeaders().get(KafkaHeaders.MESSAGE_KEY), - (String) message.getPayload(), - (Long) message.getHeaders().get(KafkaHeaders.OFFSET), - Thread.currentThread().getName(), - (Integer) message.getHeaders().get(KafkaHeaders.PARTITION_ID))); - latch.countDown(); - return addedSuccessfully; - } - - - @Override - public boolean send(Message message, long timeout) { - return send(message); - } - }); - - kafkaMessageDrivenChannelAdapter.afterPropertiesSet(); - kafkaMessageDrivenChannelAdapter.start(); - - createMessageSender("none").send(createMessagesInRange(200, 299, TEST_TOPIC, partitionCount)); - - latch.await((expectedMessageCount / 5000) + 1, TimeUnit.MINUTES); - kafkaMessageListenerContainer.stop(); - - assertThat(receivedData.valuesView().toList(), hasSize(expectedMessageCount)); - assertThat(latch.getCount(), equalTo(0L)); - System.out.println("All messages received ... checking "); - - validateMessageReceipt(receivedData, 2, partitionCount, expectedMessageCount, 1); - - // For all received messages - Collection allReceivedMessages = - Iterate.flatCollect(receivedData.keyMultiValuePairsView(), - new Function>, RichIterable>() { - - @Override - public RichIterable valueOf(Pair> object) { - return object.getTwo(); - } - - }); - - // We extract the sequence value, i.e. "Message xx" - Integer minValueInMessage = FastList.newList(allReceivedMessages) - .collect(new Function() { - @Override - public Integer valueOf(KeyedMessageWithOffset object) { - return Integer.parseInt(object.getPayload().split(" ")[1]); - } - }).min(); - - // The lowest received value is 0. That is correct, because reading started at 0 after the reset - assertThat(minValueInMessage, equalTo(0)); - } - - @Test - @SuppressWarnings("serial") - public void testLowVolumeLowConcurrencyResetAtLatest() throws Exception { - - // we will send 300 messages: first 200, then another 100 - // we will start reading from all partitions at offset 900, but since that is reset - int expectedMessageCount = 100; - int partitionCount = 5; - - createTopic(TEST_TOPIC, partitionCount, 1, 1); - - ConnectionFactory connectionFactory = getKafkaBrokerConnectionFactory(); - ArrayList readPartitions = new ArrayList(); - Map startingOffsets = new HashMap(); - for (int i = 0; i < partitionCount; i++) { - Partition partition = new Partition(TEST_TOPIC, i); - readPartitions.add(partition); - startingOffsets.put(partition, 900L); - } - - final KafkaMessageListenerContainer kafkaMessageListenerContainer = - new KafkaMessageListenerContainer(connectionFactory, - readPartitions.toArray(new Partition[readPartitions.size()])); - kafkaMessageListenerContainer.setMaxFetch(100); - kafkaMessageListenerContainer.setConcurrency(2); - final MetadataStoreOffsetManager offsetManager = - new MetadataStoreOffsetManager(connectionFactory, startingOffsets); - offsetManager.setReferenceTimestamp(-1); - final CountDownLatch offsetResetLatch = new CountDownLatch(1); - - // Manually generated proxy for catching the reset event. We only want to send data once we are - // sure that the OffsetManager has been reset. Otherwise the test may not finish (due to pollers starting after send - - OffsetManager offsetManagerWrapper = new OffsetManager() { - - @Override - public void flush() throws IOException { - offsetManager.flush(); - } - - @Override - public void close() throws IOException { - offsetManager.close(); - } - - @Override - public void deleteOffset(Partition partition) { - offsetManager.deleteOffset(partition); - } - - @Override - public void resetOffsets(Collection partitionsToReset) { - System.out.println("resetting " + offsetResetLatch.getCount() + " partitions: " - + StringUtils.collectionToDelimitedString(partitionsToReset, ",")); - offsetManager.resetOffsets(partitionsToReset); - offsetResetLatch.countDown(); - } - - @Override - public long getOffset(Partition partition) { - long offset = offsetManager.getOffset(partition); - System.out.println("Result for " + partition + " offset: " + offset); - return offset; - } - - @Override - public void updateOffset(Partition partition, long offset) { - offsetManager.updateOffset(partition, offset); - } - - }; - - kafkaMessageListenerContainer.setOffsetManager(offsetManagerWrapper); - - createMessageSender("none").send(createMessagesInRange(0, 199, TEST_TOPIC, partitionCount)); - - final MutableListMultimap receivedData = - new SynchronizedPutFastListMultimap(); - final CountDownLatch latch = new CountDownLatch(expectedMessageCount); - - KafkaMessageDrivenChannelAdapter kafkaMessageDrivenChannelAdapter = - new KafkaMessageDrivenChannelAdapter(kafkaMessageListenerContainer); - - StringDecoder decoder = new StringDecoder(); - kafkaMessageDrivenChannelAdapter.setKeyDecoder(decoder); - kafkaMessageDrivenChannelAdapter.setPayloadDecoder(decoder); - kafkaMessageDrivenChannelAdapter.setBeanFactory(mock(BeanFactory.class)); - kafkaMessageDrivenChannelAdapter.setOutputChannel(new MessageChannel() { - @Override - public boolean send(Message message) { - boolean addedSuccessfully = receivedData.put( - (Integer) message.getHeaders().get(KafkaHeaders.PARTITION_ID), - new KeyedMessageWithOffset( - (String) message.getHeaders().get(KafkaHeaders.MESSAGE_KEY), - (String) message.getPayload(), - (Long) message.getHeaders().get(KafkaHeaders.OFFSET), - Thread.currentThread().getName(), - (Integer) message.getHeaders().get(KafkaHeaders.PARTITION_ID))); - latch.countDown(); - return addedSuccessfully; - } - - @Override - public boolean send(Message message, long timeout) { - return send(message); - } - }); - - kafkaMessageDrivenChannelAdapter.afterPropertiesSet(); - kafkaMessageDrivenChannelAdapter.start(); - - // we wait for the reset event first - offsetResetLatch.await(1000, TimeUnit.MILLISECONDS); - Thread.sleep(1000); - createMessageSender("none").send(createMessagesInRange(200, 299, TEST_TOPIC, partitionCount)); - - latch.await((expectedMessageCount / 5000) + 1, TimeUnit.MINUTES); - kafkaMessageListenerContainer.stop(); - - assertThat(receivedData.valuesView().toList(), hasSize(expectedMessageCount)); - assertThat(latch.getCount(), equalTo(0L)); - System.out.println("All messages received ... checking "); - - validateMessageReceipt(receivedData, 2, partitionCount, expectedMessageCount, 1); - - // For all received messages - Collection allReceivedMessages = - Iterate.flatCollect(receivedData.keyMultiValuePairsView(), - new Function>, RichIterable>() { - - @Override - public RichIterable valueOf(Pair> object) { - return object.getTwo(); - } - - }); - - // We extract the sequence value, i.e. "Message xx" - Integer minValueInMessage = FastList.newList(allReceivedMessages) - .collect(new Function() { - - @Override - public Integer valueOf(KeyedMessageWithOffset object) { - return Integer.parseInt(object.getPayload().split(" ")[1]); - } - - }) - .min(); - - // The lowest received value is 0. That is correct, because reading started at offset 40 (last after the first - // 200 messages have been sent) after the reset - assertThat(minValueInMessage, equalTo(200)); - } - -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker20Tests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker20Tests.java deleted file mode 100644 index dbefac9fb3..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker20Tests.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package org.springframework.integration.kafka.listener; - -import org.junit.Ignore; -import org.junit.Rule; -import org.junit.Test; - -import org.springframework.integration.kafka.rule.KafkaEmbedded; - -/** - * @author Marius Bogoevici - */ -public class MultiBroker20Tests extends AbstractMessageListenerContainerTests { - - @Rule - public final KafkaEmbedded kafkaEmbeddedBrokerRule = new KafkaEmbedded(20); - - @Override - public KafkaEmbedded getKafkaRule() { - return kafkaEmbeddedBrokerRule; - } - - @Test - @Ignore - public void testLowVolumeHighConcurrency() throws Exception { - createTopic(TEST_TOPIC, 100, 20, 1); - runMessageListenerTest(100, 20, 100, 1000, 1, "none", TEST_TOPIC); - } - - @Test - @Ignore - public void testMediumVolumeHighConcurrency() throws Exception { - createTopic(TEST_TOPIC, 100, 20, 1); - runMessageListenerTest(100, 20, 100, 10000, 1, "none", TEST_TOPIC); - } - - @Test - @Ignore - public void testHighVolumeHighConcurrency() throws Exception { - createTopic(TEST_TOPIC, 100, 20, 1); - runMessageListenerTest(100, 20, 100, 100000, 1, "none", TEST_TOPIC); - } - -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker2Tests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker2Tests.java deleted file mode 100644 index 3974f3c356..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker2Tests.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package org.springframework.integration.kafka.listener; - -import org.junit.Ignore; -import org.junit.Rule; -import org.junit.Test; - -import org.springframework.integration.kafka.rule.KafkaEmbedded; -import org.springframework.integration.kafka.rule.KafkaRule; - -/** - * @author Marius Bogoevici - */ -public class MultiBroker2Tests extends AbstractMessageListenerContainerTests { - - @Rule - public KafkaEmbedded kafkaEmbeddedBrokerRule = new KafkaEmbedded(2); - - @Override - public KafkaRule getKafkaRule() { - return kafkaEmbeddedBrokerRule; - } - - @Test - public void testLowVolumeLowConcurrency() throws Exception { - createTopic(TEST_TOPIC, 5, 2, 1); - runMessageListenerTest(100, 2, 5, 100, 1, "none", TEST_TOPIC); - } - - @Test - @Ignore - public void testMediumVolumeLowConcurrency() throws Exception { - createTopic(TEST_TOPIC, 5, 2, 1); - runMessageListenerTest(100, 2, 5, 1000, 1, "none", TEST_TOPIC); - } - - @Test - @Ignore - public void testHighVolumeLowConcurrency() throws Exception { - createTopic(TEST_TOPIC, 5, 2, 1); - runMessageListenerTest(100, 2, 5, 10000, 1, "none", TEST_TOPIC); - } - - @Test - @Ignore - public void testLowVolumeMediumConcurrency() throws Exception { - createTopic(TEST_TOPIC, 5, 2, 1); - runMessageListenerTest(100, 5, 5, 100, 1, "none", TEST_TOPIC); - } - - @Test - @Ignore - public void testMediumVolumeMediumConcurrency() throws Exception { - createTopic(TEST_TOPIC, 5, 2, 1); - runMessageListenerTest(100, 5, 5, 1000, 1, "none", TEST_TOPIC); - } - - @Test - @Ignore - public void testHighVolumeMediumConcurrency() throws Exception { - createTopic(TEST_TOPIC, 5, 2, 1); - runMessageListenerTest(100, 5, 5, 100000, 1, "none", TEST_TOPIC); - } - - - @Test - @Ignore - public void testLowVolumeHighConcurrency() throws Exception { - createTopic(TEST_TOPIC, 100, 2, 1); - runMessageListenerTest(100, 20, 100, 1000, 1, "none", TEST_TOPIC); - } - - @Test - @Ignore - public void testMediumVolumeHighConcurrency() throws Exception { - createTopic(TEST_TOPIC, 100, 2, 1); - runMessageListenerTest(100, 20, 100, 10000, 1, "none", TEST_TOPIC); - } - - @Test - @Ignore - public void testHighVolumeHighConcurrency() throws Exception { - createTopic(TEST_TOPIC, 100, 2, 1); - runMessageListenerTest(100, 20, 100, 100000, 1, "none", TEST_TOPIC); - } - -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker5ReplicatedTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker5ReplicatedTests.java deleted file mode 100644 index f0f2728828..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker5ReplicatedTests.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package org.springframework.integration.kafka.listener; - -import org.junit.Ignore; -import org.junit.Rule; -import org.junit.Test; - -import org.springframework.integration.kafka.rule.KafkaEmbedded; - -/** - * @author Marius Bogoevici - */ -public class MultiBroker5ReplicatedTests extends AbstractMessageListenerContainerTests { - - @Rule - public KafkaEmbedded kafkaEmbeddedBrokerRule = new KafkaEmbedded(5); - - @Override - public KafkaEmbedded getKafkaRule() { - return kafkaEmbeddedBrokerRule; - } - - @Test - public void testLowVolumeLowConcurrency() throws Exception { - createTopic(TEST_TOPIC, 5, 5, 3); - runMessageListenerTest(100, 2, 5, 100, 1, "none", TEST_TOPIC); - } - - @Test - @Ignore - public void testMediumVolumeLowConcurrency() throws Exception { - createTopic(TEST_TOPIC, 5, 5, 3); - runMessageListenerTest(100, 2, 5, 1000, 1, "none", TEST_TOPIC); - } - - @Test - @Ignore - public void testHighVolumeLowConcurrency() throws Exception { - createTopic(TEST_TOPIC, 5, 5, 3); - runMessageListenerTest(100, 2, 5, 10000, 1, "none", TEST_TOPIC); - } - - @Test - @Ignore - public void testLowVolumeMediumConcurrency() throws Exception { - createTopic(TEST_TOPIC, 5, 5, 3); - runMessageListenerTest(100, 5, 5, 100, 1, "none", TEST_TOPIC); - } - - @Test - @Ignore - public void testMediumVolumeMediumConcurrency() throws Exception { - createTopic(TEST_TOPIC, 5, 5, 3); - runMessageListenerTest(100, 5, 5, 1000, 1, "none", TEST_TOPIC); - } - - @Test - @Ignore - public void testHighVolumeMediumConcurrency() throws Exception { - createTopic(TEST_TOPIC, 5, 5, 1); - runMessageListenerTest(100, 5, 5, 100000, 1, "none", TEST_TOPIC); - } - - - @Test - @Ignore - public void testLowVolumeHighConcurrency() throws Exception { - createTopic(TEST_TOPIC, 100, 5, 3); - runMessageListenerTest(100, 20, 100, 1000, 1, "none", TEST_TOPIC); - } - - @Test - @Ignore - public void testMediumVolumeHighConcurrency() throws Exception { - createTopic(TEST_TOPIC, 100, 5, 3); - runMessageListenerTest(100, 20, 100, 10000, 1, "none", TEST_TOPIC); - } - - @Test - @Ignore - public void testHighVolumeHighConcurrency() throws Exception { - createTopic(TEST_TOPIC, 100, 5, 3); - runMessageListenerTest(100, 20, 100, 100000, 1, "none", TEST_TOPIC); - } - -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker5ReplicatedWithBounceTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker5ReplicatedWithBounceTests.java deleted file mode 100644 index 7aa1fae903..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker5ReplicatedWithBounceTests.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package org.springframework.integration.kafka.listener; - -import static org.hamcrest.collection.IsCollectionWithSize.hasSize; -import static org.hamcrest.core.IsEqual.equalTo; -import static org.junit.Assert.assertThat; -import static org.springframework.integration.kafka.util.MessageUtils.decodeKey; -import static org.springframework.integration.kafka.util.MessageUtils.decodePayload; - -import java.util.ArrayList; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import com.gs.collections.api.multimap.list.MutableListMultimap; -import com.gs.collections.impl.multimap.list.SynchronizedPutFastListMultimap; - -import kafka.serializer.StringDecoder; -import kafka.utils.VerifiableProperties; - -import org.junit.Ignore; -import org.junit.Rule; -import org.junit.Test; - -import org.springframework.integration.kafka.core.ConnectionFactory; -import org.springframework.integration.kafka.core.KafkaMessage; -import org.springframework.integration.kafka.core.Partition; -import org.springframework.integration.kafka.rule.KafkaEmbedded; - -/** - * @author Marius Bogoevici - */ -@Ignore -public class MultiBroker5ReplicatedWithBounceTests extends AbstractMessageListenerContainerTests { - - @Rule - public KafkaEmbedded kafkaEmbeddedBrokerRule = new KafkaEmbedded(5); - - @Override - public KafkaEmbedded getKafkaRule() { - return kafkaEmbeddedBrokerRule; - } - - @Test - public void testLowVolumeLowConcurrency() throws Exception { - int partitionCount = 5; - - createTopic(TEST_TOPIC, partitionCount, 5, 3); - - Thread.sleep(1000); - - ConnectionFactory connectionFactory = getKafkaBrokerConnectionFactory(); - ArrayList readPartitions = new ArrayList(); - for (int i = 0; i < partitionCount; i++) { - readPartitions.add(new Partition(TEST_TOPIC, i)); - } - final KafkaMessageListenerContainer kafkaMessageListenerContainer = - new KafkaMessageListenerContainer(connectionFactory, - readPartitions.toArray(new Partition[readPartitions.size()])); - kafkaMessageListenerContainer.setMaxFetch(100); - kafkaMessageListenerContainer.setConcurrency(2); - - final int expectedMessageCount = 100; - - createMessageSender("none").send(createMessages(100, TEST_TOPIC, partitionCount)); - - - final MutableListMultimap receivedData = - new SynchronizedPutFastListMultimap(); - final CountDownLatch latch = new CountDownLatch(expectedMessageCount); - kafkaMessageListenerContainer.setMessageListener(new MessageListener() { - - @Override - public void onMessage(KafkaMessage message) { - StringDecoder decoder = new StringDecoder(new VerifiableProperties()); - receivedData.put(message.getMetadata().getPartition().getId(), - new KeyedMessageWithOffset(decodeKey(message, decoder), decodePayload(message, decoder), - message.getMetadata().getOffset(), Thread.currentThread().getName(), - message.getMetadata().getPartition().getId())); - latch.countDown(); - } - - }); - - kafkaMessageListenerContainer.start(); - - for (int i = 0; i < 5; i++) { - if (i % 2 == 0) { - kafkaEmbeddedBrokerRule.bounce(i); - } - } - - latch.await(120, TimeUnit.SECONDS); - kafkaMessageListenerContainer.stop(); - - assertThat(receivedData.valuesView().toList(), hasSize(expectedMessageCount)); - assertThat(latch.getCount(), equalTo(0L)); - System.out.println("All messages received ... checking "); - - validateMessageReceipt(receivedData, 2, partitionCount, expectedMessageCount, 1); - - } - -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker5Tests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker5Tests.java deleted file mode 100644 index 1d0bea2784..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/MultiBroker5Tests.java +++ /dev/null @@ -1,102 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package org.springframework.integration.kafka.listener; - -import org.junit.Ignore; -import org.junit.Rule; -import org.junit.Test; - -import org.springframework.integration.kafka.rule.KafkaEmbedded; - -/** - * @author Marius Bogoevici - */ -public class MultiBroker5Tests extends AbstractMessageListenerContainerTests { - - @Rule - public KafkaEmbedded kafkaEmbeddedBrokerRule = new KafkaEmbedded(5); - - @Override - public KafkaEmbedded getKafkaRule() { - return kafkaEmbeddedBrokerRule; - } - - @Test - public void testLowVolumeLowConcurrency() throws Exception { - createTopic(TEST_TOPIC, 5, 5, 1); - runMessageListenerTest(100, 2, 5, 100, 1, "none", TEST_TOPIC); - } - - @Test - @Ignore - public void testMediumVolumeLowConcurrency() throws Exception { - createTopic(TEST_TOPIC, 5, 5, 1); - runMessageListenerTest(100, 2, 5, 1000, 1, "none", TEST_TOPIC); - } - - @Test - @Ignore - public void testHighVolumeLowConcurrency() throws Exception { - createTopic(TEST_TOPIC, 5, 5, 1); - runMessageListenerTest(100, 2, 5, 10000, 1, "none", TEST_TOPIC); - } - - @Test - @Ignore - public void testLowVolumeMediumConcurrency() throws Exception { - createTopic(TEST_TOPIC, 5, 5, 1); - runMessageListenerTest(100, 5, 5, 100, 1, "none", TEST_TOPIC); - } - - @Test - @Ignore - public void testMediumVolumeMediumConcurrency() throws Exception { - createTopic(TEST_TOPIC, 5, 5, 1); - runMessageListenerTest(100, 5, 5, 1000, 1, "none", TEST_TOPIC); - } - - @Test - @Ignore - public void testHighVolumeMediumConcurrency() throws Exception { - createTopic(TEST_TOPIC, 5, 5, 1); - runMessageListenerTest(100, 5, 5, 100000, 1, "none", TEST_TOPIC); - } - - - @Test - @Ignore - public void testLowVolumeHighConcurrency() throws Exception { - createTopic(TEST_TOPIC, 100, 5, 1); - runMessageListenerTest(100, 20, 100, 1000, 1, "none", TEST_TOPIC); - } - - @Test - @Ignore - public void testMediumVolumeHighConcurrency() throws Exception { - createTopic(TEST_TOPIC, 100, 5, 1); - runMessageListenerTest(100, 20, 100, 10000, 1, "none", TEST_TOPIC); - } - - @Test - @Ignore - public void testHighVolumeHighConcurrency() throws Exception { - createTopic(TEST_TOPIC, 100, 5, 1); - runMessageListenerTest(100, 20, 100, 100000, 1, "none", TEST_TOPIC); - } - -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/NewBrokerRecoveryTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/NewBrokerRecoveryTests.java deleted file mode 100644 index 9046078e5f..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/NewBrokerRecoveryTests.java +++ /dev/null @@ -1,136 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package org.springframework.integration.kafka.listener; - -import static org.hamcrest.collection.IsCollectionWithSize.hasSize; -import static org.hamcrest.core.IsEqual.equalTo; -import static org.junit.Assert.assertThat; -import static org.springframework.integration.kafka.util.MessageUtils.decodeKey; -import static org.springframework.integration.kafka.util.MessageUtils.decodePayload; - -import java.util.ArrayList; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import com.gs.collections.api.multimap.list.MutableListMultimap; -import com.gs.collections.impl.multimap.list.SynchronizedPutFastListMultimap; -import kafka.serializer.StringDecoder; -import kafka.utils.VerifiableProperties; -import org.junit.Rule; -import org.junit.Test; - -import org.springframework.integration.kafka.core.BrokerAddressListConfiguration; -import org.springframework.integration.kafka.core.ConnectionFactory; -import org.springframework.integration.kafka.core.DefaultConnectionFactory; -import org.springframework.integration.kafka.core.KafkaMessage; -import org.springframework.integration.kafka.core.Partition; -import org.springframework.integration.kafka.rule.KafkaEmbedded; - -/** - * @author Marius Bogoevici - */ - -public class NewBrokerRecoveryTests extends AbstractMessageListenerContainerTests { - - @Rule - public KafkaEmbedded kafkaEmbeddedBrokerRule = new KafkaEmbedded(2); - - @Override - public KafkaEmbedded getKafkaRule() { - return kafkaEmbeddedBrokerRule; - } - - @Test - public void testCompleteShutdown() throws Exception { - int partitionCount = 1; - - createTopic(TEST_TOPIC, partitionCount, 2, 2); - - // stop one Kafka instance - the partition is now underreplicated and only one - kafkaEmbeddedBrokerRule.bounce(0); - - // connect to the live broker only - ConnectionFactory connectionFactory = - new DefaultConnectionFactory(new BrokerAddressListConfiguration(kafkaEmbeddedBrokerRule.getBrokerAddress(1))); - ArrayList readPartitions = new ArrayList<>(); - readPartitions.add(new Partition(TEST_TOPIC, 0)); - final KafkaMessageListenerContainer kafkaMessageListenerContainer = - new KafkaMessageListenerContainer(connectionFactory, - readPartitions.toArray(new Partition[readPartitions.size()])); - kafkaMessageListenerContainer.setMaxFetch(100); - kafkaMessageListenerContainer.setConcurrency(1); - - final int expectedMessageCount = 200; - createMessageSender("none", 1).send(createMessagesInRange(0, 49, TEST_TOPIC, partitionCount)); - - final MutableListMultimap receivedData = - new SynchronizedPutFastListMultimap(); - final CountDownLatch latch = new CountDownLatch(expectedMessageCount); - kafkaMessageListenerContainer.setMessageListener(new MessageListener() { - - @Override - public void onMessage(KafkaMessage message) { - StringDecoder decoder = new StringDecoder(new VerifiableProperties()); - receivedData.put(message.getMetadata().getPartition().getId(), - new KeyedMessageWithOffset(decodeKey(message, decoder), decodePayload(message, decoder), - message.getMetadata().getOffset(), Thread.currentThread().getName(), - message.getMetadata().getPartition().getId())); - latch.countDown(); - } - - }); - - - kafkaMessageListenerContainer.start(); - - // now start sending messages again - createMessageSender("none", 1).send(createMessagesInRange(50, 99, TEST_TOPIC, partitionCount)); - - // start the other Kafka instance - kafkaEmbeddedBrokerRule.restart(0); - // sleep to let the brokers sync up - kafkaEmbeddedBrokerRule.waitUntilSynced(TEST_TOPIC, 0); - // bounce the other server - kafkaEmbeddedBrokerRule.bounce(1); - - // now start sending messages again - createMessageSender("none", 0).send(createMessagesInRange(100, 149, TEST_TOPIC, partitionCount)); - - // stop the other Kafka instance - kafkaEmbeddedBrokerRule.restart(1); - // sleep to let the brokers sync up - kafkaEmbeddedBrokerRule.waitUntilSynced(TEST_TOPIC, 1); - - // bounce the other server - kafkaEmbeddedBrokerRule.bounce(0); - - // now start sending messages again - createMessageSender("none", 1).send(createMessagesInRange(149, 199, TEST_TOPIC, partitionCount)); - - - latch.await(50, TimeUnit.SECONDS); - kafkaMessageListenerContainer.stop(); - - assertThat(receivedData.valuesView().toList(), hasSize(expectedMessageCount)); - assertThat(latch.getCount(), equalTo(0L)); - - validateMessageReceipt(receivedData, 1, partitionCount, expectedMessageCount, 1); - - } - -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/SingleBrokerExternalTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/SingleBrokerExternalTests.java deleted file mode 100644 index 43455b3f92..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/SingleBrokerExternalTests.java +++ /dev/null @@ -1,131 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package org.springframework.integration.kafka.listener; - -import java.util.UUID; - -import org.junit.After; -import org.junit.ClassRule; -import org.junit.Ignore; -import org.junit.Test; - -import org.springframework.integration.kafka.rule.KafkaRule; -import org.springframework.integration.kafka.rule.KafkaRunning; - -/** - * @author Marius Bogoevici - */ -@Ignore -public class SingleBrokerExternalTests extends AbstractMessageListenerContainerTests { - - @ClassRule - public static final KafkaRunning kafkaRunningRule = KafkaRunning.isRunning(); - - @Override - public KafkaRule getKafkaRule() { - return kafkaRunningRule; - } - - @Override - @After - public void cleanUp() { - // do nothing, each topic must be created individually - } - - private String generateTopicName() { - return "test" + UUID.randomUUID().toString().replace("-",""); - } - - @Test - public void testLowVolumeLowConcurrency() throws Exception { - String topicName = generateTopicName(); - createTopic(topicName, 5, 1, 1); - runMessageListenerTest(100, 2, 5, 100, 1, "none", topicName); - deleteTopic(topicName); - } - - @Test - public void testMediumVolumeLowConcurrency() throws Exception { - String topicName = generateTopicName(); - createTopic(topicName, 5, 1, 1); - runMessageListenerTest(100, 2, 5, 1000, 1, "none", topicName); - deleteTopic(topicName); - } - - @Test - @Ignore - public void testHighVolumeLowConcurrency() throws Exception { - String topicName = generateTopicName(); - createTopic(topicName, 5, 1, 1); - runMessageListenerTest(100, 2, 5, 10000, 1, "none", topicName); - deleteTopic(topicName); - } - - @Test - public void testLowVolumeMediumConcurrency() throws Exception { - String topicName = generateTopicName(); - createTopic(topicName, 5, 1, 1); - runMessageListenerTest(100, 5, 5, 100, 1, "none", topicName); - deleteTopic(topicName); - } - - @Test - public void testMediumVolumeMediumConcurrency() throws Exception { - String topicName = generateTopicName(); - createTopic(topicName, 5, 1, 1); - runMessageListenerTest(100, 5, 5, 1000, 1, "none", topicName); - deleteTopic(topicName); - } - - @Test - @Ignore - public void testHighVolumeMediumConcurrency() throws Exception { - String topicName = generateTopicName(); - createTopic(topicName, 5, 1, 1); - runMessageListenerTest(100, 5, 5, 100000, 1, "none", topicName); - deleteTopic(topicName); - } - - @Test - @Ignore - public void testLowVolumeHighConcurrency() throws Exception { - String topicName = generateTopicName(); - createTopic(topicName, 100, 1, 1); - runMessageListenerTest(100, 20, 100, 1000, 1, "none", topicName); - deleteTopic(topicName); - } - - @Test - @Ignore - public void testMediumVolumeHighConcurrency() throws Exception { - String topicName = generateTopicName(); - createTopic(topicName, 100, 1, 1); - runMessageListenerTest(100, 20, 100, 10000, 1, "none", topicName); - deleteTopic(topicName); - } - - @Test - @Ignore - public void testHighVolumeHighConcurrency() throws Exception { - String topicName = generateTopicName(); - createTopic(topicName, 100, 1, 1); - runMessageListenerTest(100, 20, 100, 100000, 1, "none", topicName); - deleteTopic(topicName); - } - -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/SingleBrokerRecoveryTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/SingleBrokerRecoveryTests.java deleted file mode 100644 index a262aa1186..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/SingleBrokerRecoveryTests.java +++ /dev/null @@ -1,118 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package org.springframework.integration.kafka.listener; - -import static org.hamcrest.collection.IsCollectionWithSize.hasSize; -import static org.hamcrest.core.IsEqual.equalTo; -import static org.junit.Assert.assertThat; -import static org.springframework.integration.kafka.util.MessageUtils.decodeKey; -import static org.springframework.integration.kafka.util.MessageUtils.decodePayload; - -import java.util.ArrayList; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import com.gs.collections.api.multimap.list.MutableListMultimap; -import com.gs.collections.impl.multimap.list.SynchronizedPutFastListMultimap; -import kafka.serializer.StringDecoder; -import kafka.utils.VerifiableProperties; -import org.junit.Rule; -import org.junit.Test; - -import org.springframework.integration.kafka.core.ConnectionFactory; -import org.springframework.integration.kafka.core.KafkaMessage; -import org.springframework.integration.kafka.core.Partition; -import org.springframework.integration.kafka.rule.KafkaEmbedded; - -/** - * @author Marius Bogoevici - */ - -public class SingleBrokerRecoveryTests extends AbstractMessageListenerContainerTests { - - @Rule - public KafkaEmbedded kafkaEmbeddedBrokerRule = new KafkaEmbedded(1); - - @Override - public KafkaEmbedded getKafkaRule() { - return kafkaEmbeddedBrokerRule; - } - - @Test - public void testCompleteShutdown() throws Exception { - int partitionCount = 1; - - createTopic(TEST_TOPIC, partitionCount, 1, 1); - - ConnectionFactory connectionFactory = getKafkaBrokerConnectionFactory(); - ArrayList readPartitions = new ArrayList(); - readPartitions.add(new Partition(TEST_TOPIC, 0)); - final KafkaMessageListenerContainer kafkaMessageListenerContainer = - new KafkaMessageListenerContainer(connectionFactory, - readPartitions.toArray(new Partition[readPartitions.size()])); - kafkaMessageListenerContainer.setMaxFetch(100); - kafkaMessageListenerContainer.setConcurrency(1); - - final int expectedMessageCount = 100; - - createMessageSender("none").send(createMessages(10, TEST_TOPIC,partitionCount)); - - final MutableListMultimap receivedData = - new SynchronizedPutFastListMultimap(); - final CountDownLatch latch = new CountDownLatch(expectedMessageCount); - kafkaMessageListenerContainer.setMessageListener(new MessageListener() { - - @Override - public void onMessage(KafkaMessage message) { - StringDecoder decoder = new StringDecoder(new VerifiableProperties()); - receivedData.put(message.getMetadata().getPartition().getId(), - new KeyedMessageWithOffset(decodeKey(message, decoder), decodePayload(message, decoder), - message.getMetadata().getOffset(), Thread.currentThread().getName(), - message.getMetadata().getPartition().getId())); - latch.countDown(); - } - - }); - - - kafkaMessageListenerContainer.start(); - - // stop Kafka - kafkaEmbeddedBrokerRule.bounce(0, false); - - // sleep one second to let things settle - Thread.sleep(1000); - - // restart Kafka - kafkaEmbeddedBrokerRule.restart(0); - kafkaEmbeddedBrokerRule.waitUntilSynced(TEST_TOPIC,0); - - // now start sending messages again - createMessageSender("none").send(createMessages(90, TEST_TOPIC,partitionCount)); - - latch.await(50, TimeUnit.SECONDS); - kafkaMessageListenerContainer.stop(); - - assertThat(receivedData.valuesView().toList(), hasSize(expectedMessageCount)); - assertThat(latch.getCount(), equalTo(0L)); - - validateMessageReceipt(receivedData, 1, partitionCount, expectedMessageCount, 1); - - } - -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/SingleBrokerTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/SingleBrokerTests.java deleted file mode 100644 index 031cc1efa0..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/SingleBrokerTests.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package org.springframework.integration.kafka.listener; - -import org.junit.Ignore; -import org.junit.Rule; -import org.junit.Test; - -import org.springframework.integration.kafka.rule.KafkaEmbedded; - -/** - * @author Marius Bogoevici - */ -public class SingleBrokerTests extends AbstractMessageListenerContainerTests { - - @Rule - public final KafkaEmbedded kafkaEmbeddedBrokerRule = new KafkaEmbedded(1); - - @Override - public KafkaEmbedded getKafkaRule() { - return kafkaEmbeddedBrokerRule; - } - - @Test - public void testLowVolumeLowConcurrency() throws Exception { - createTopic(TEST_TOPIC, 5, 1, 1); - runMessageListenerTest(100, 2, 5, 100, 1, "none", TEST_TOPIC); - } - - @Test - public void testMediumVolumeLowConcurrency() throws Exception { - createTopic(TEST_TOPIC, 5, 1, 1); - runMessageListenerTest(100, 2, 5, 1000, 1, "none", TEST_TOPIC); - } - - @Test - @Ignore - public void testHighVolumeLowConcurrency() throws Exception { - createTopic(TEST_TOPIC, 5, 1, 1); - runMessageListenerTest(100, 2, 5, 10000, 1, "none", TEST_TOPIC); - } - - @Test - public void testLowVolumeMediumConcurrency() throws Exception { - createTopic(TEST_TOPIC, 5, 1, 1); - runMessageListenerTest(100, 5, 5, 100, 1, "none", TEST_TOPIC); - } - - @Test - public void testMediumVolumeMediumConcurrency() throws Exception { - createTopic(TEST_TOPIC, 5, 1, 1); - runMessageListenerTest(100, 5, 5, 1000, 1, "none", TEST_TOPIC); - } - - @Test - @Ignore - public void testHighVolumeMediumConcurrency() throws Exception { - createTopic(TEST_TOPIC, 5, 1, 1); - runMessageListenerTest(100, 5, 5, 100000, 1, "none", TEST_TOPIC); - } - - @Test - @Ignore - public void testLowVolumeHighConcurrency() throws Exception { - createTopic(TEST_TOPIC, 100, 1, 1); - runMessageListenerTest(100, 20, 100, 1000, 1, "none", TEST_TOPIC); - } - - @Test - @Ignore - public void testMediumVolumeHighConcurrency() throws Exception { - createTopic(TEST_TOPIC, 100, 1, 1); - runMessageListenerTest(100, 20, 100, 10000, 1, "none", TEST_TOPIC); - } - - @Test - @Ignore - public void testHighVolumeHighConcurrency() throws Exception { - createTopic(TEST_TOPIC, 100, 1, 1); - runMessageListenerTest(100, 20, 100, 100000, 1, "none", TEST_TOPIC); - } - -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/SingleBrokerWithCompressionTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/SingleBrokerWithCompressionTests.java deleted file mode 100644 index 8c448610b3..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/SingleBrokerWithCompressionTests.java +++ /dev/null @@ -1,98 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package org.springframework.integration.kafka.listener; - -import org.junit.Ignore; -import org.junit.Rule; -import org.junit.Test; - -import org.springframework.integration.kafka.rule.KafkaEmbedded; - -/** - * @author Marius Bogoevici - */ -public class SingleBrokerWithCompressionTests extends AbstractMessageListenerContainerTests { - - @Rule - public final KafkaEmbedded kafkaEmbeddedBrokerRule = new KafkaEmbedded(1); - - @Override - public KafkaEmbedded getKafkaRule() { - return kafkaEmbeddedBrokerRule; - } - - @Test - public void testLowVolumeLowConcurrency() throws Exception { - createTopic(TEST_TOPIC, 5, 1, 1); - runMessageListenerTest(10000, 2, 5, 100, 1, "gzip", TEST_TOPIC); - } - - @Test - public void testMediumVolumeLowConcurrency() throws Exception { - createTopic(TEST_TOPIC, 5, 1, 1); - runMessageListenerTest(10000, 2, 5, 1000, 1, "gzip", TEST_TOPIC); - } - - @Test - @Ignore - public void testHighVolumeLowConcurrency() throws Exception { - createTopic(TEST_TOPIC, 5, 1, 1); - runMessageListenerTest(10000, 2, 5, 10000, 1, "gzip", TEST_TOPIC); - } - - @Test - public void testLowVolumeMediumConcurrency() throws Exception { - createTopic(TEST_TOPIC, 5, 1, 1); - runMessageListenerTest(10000, 5, 5, 100, 1, "gzip", TEST_TOPIC); - } - - @Test - public void testMediumVolumeMediumConcurrency() throws Exception { - createTopic(TEST_TOPIC, 5, 1, 1); - runMessageListenerTest(10000, 5, 5, 1000, 1, "gzip", TEST_TOPIC); - } - - @Test - @Ignore - public void testHighVolumeMediumConcurrency() throws Exception { - createTopic(TEST_TOPIC, 5, 1, 1); - runMessageListenerTest(10000, 5, 5, 100000, 1, "gzip", TEST_TOPIC); - } - - @Test - @Ignore - public void testLowVolumeHighConcurrency() throws Exception { - createTopic(TEST_TOPIC, 100, 1, 1); - runMessageListenerTest(10000, 20, 100, 1000, 1, "gzip", TEST_TOPIC); - } - - @Test - @Ignore - public void testMediumVolumeHighConcurrency() throws Exception { - createTopic(TEST_TOPIC, 100, 1, 1); - runMessageListenerTest(10000, 20, 100, 10000, 1, "gzip", TEST_TOPIC); - } - - @Test - @Ignore - public void testHighVolumeHighConcurrency() throws Exception { - createTopic(TEST_TOPIC, 100, 1, 1); - runMessageListenerTest(10000, 20, 100, 100000, 1, "gzip", TEST_TOPIC); - } - -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/SingleBrokerWithManualAckTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/SingleBrokerWithManualAckTests.java deleted file mode 100644 index af553da8fe..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/SingleBrokerWithManualAckTests.java +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.listener; - -import static org.hamcrest.CoreMatchers.nullValue; -import static org.hamcrest.collection.IsCollectionWithSize.hasSize; -import static org.hamcrest.core.IsEqual.equalTo; -import static org.junit.Assert.assertThat; -import static org.springframework.integration.kafka.util.MessageUtils.decodeKey; -import static org.springframework.integration.kafka.util.MessageUtils.decodePayload; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import org.junit.Rule; -import org.junit.Test; - -import org.springframework.integration.kafka.core.ConnectionFactory; -import org.springframework.integration.kafka.core.KafkaMessage; -import org.springframework.integration.kafka.core.Partition; -import org.springframework.integration.kafka.rule.KafkaEmbedded; -import org.springframework.integration.metadata.SimpleMetadataStore; - -import com.gs.collections.api.multimap.list.MutableListMultimap; -import com.gs.collections.impl.list.mutable.FastList; -import com.gs.collections.impl.multimap.list.SynchronizedPutFastListMultimap; - -import kafka.serializer.StringDecoder; -import kafka.utils.VerifiableProperties; - -/** - * @author Marius Bogoevici - */ -public class SingleBrokerWithManualAckTests extends AbstractMessageListenerContainerTests { - - @Rule - public final KafkaEmbedded kafkaEmbeddedBrokerRule = new KafkaEmbedded(1); - - @Override - public KafkaEmbedded getKafkaRule() { - return kafkaEmbeddedBrokerRule; - } - - @Test(expected = IllegalArgumentException.class) - public void testMessageListenerRequiredIfAutoAckFail() throws Exception { - createTopic(TEST_TOPIC, 5, 1, 1); - ConnectionFactory connectionFactory = getKafkaBrokerConnectionFactory(); - ArrayList readPartitions = new ArrayList(); - for (int i = 0; i < 5; i++) { - readPartitions.add(new Partition(TEST_TOPIC, i)); - } - final KafkaMessageListenerContainer kafkaMessageListenerContainer = new KafkaMessageListenerContainer( - connectionFactory, readPartitions.toArray(new Partition[readPartitions.size()])); - kafkaMessageListenerContainer.setMaxFetch(100); - kafkaMessageListenerContainer.setConcurrency(2); - - kafkaMessageListenerContainer.setMessageListener(new Object()); - } - - @Test - public void testLowVolumeLowConcurrency() throws Exception { - int partitionCount = 5; - - createTopic(TEST_TOPIC, partitionCount, 1, 1); - - ConnectionFactory connectionFactory = getKafkaBrokerConnectionFactory(); - ArrayList readPartitions = new ArrayList(); - for (int i = 0; i < partitionCount; i++) { - if (i % 1 == 0) { - readPartitions.add(new Partition(TEST_TOPIC, i)); - } - } - final KafkaMessageListenerContainer kafkaMessageListenerContainer = new KafkaMessageListenerContainer( - connectionFactory, readPartitions.toArray(new Partition[readPartitions.size()])); - kafkaMessageListenerContainer.setMaxFetch(100); - kafkaMessageListenerContainer.setConcurrency(2); - MetadataStoreOffsetManager offsetManager = new MetadataStoreOffsetManager(connectionFactory); - SimpleMetadataStore metadataStore = new SimpleMetadataStore(); - offsetManager.setMetadataStore(metadataStore); - kafkaMessageListenerContainer.setOffsetManager(offsetManager); - - int expectedMessageCount = 100; - - final List acknowledgments = Collections.synchronizedList(new ArrayList()); - - final MutableListMultimap receivedData = - new SynchronizedPutFastListMultimap(); - final CountDownLatch latch = new CountDownLatch(expectedMessageCount); - kafkaMessageListenerContainer.setMessageListener(new AcknowledgingMessageListener() { - @Override - public void onMessage(KafkaMessage message, Acknowledgment acknowledgment) { - StringDecoder decoder = new StringDecoder(new VerifiableProperties()); - receivedData.put(message.getMetadata().getPartition().getId(), - new KeyedMessageWithOffset(decodeKey(message, decoder), decodePayload(message, decoder), - message.getMetadata().getOffset(), Thread.currentThread().getName(), message - .getMetadata().getPartition().getId())); - acknowledgments.add(acknowledgment); - latch.countDown(); - } - }); - - kafkaMessageListenerContainer.start(); - - createMessageSender("none").send(createMessages(100, TEST_TOPIC, partitionCount)); - - latch.await((expectedMessageCount / 5000) + 1, TimeUnit.MINUTES); - kafkaMessageListenerContainer.stop(); - - assertThat(receivedData.valuesView().toList(), hasSize(expectedMessageCount)); - assertThat(latch.getCount(), equalTo(0L)); - System.out.println("All messages received ... checking "); - - validateMessageReceipt(receivedData, 2, partitionCount, expectedMessageCount, 1); - - // at this point, all messages have been processed but not acknowledged - for (Partition readPartition : readPartitions) { - assertThat(metadataStore.get(offsetManager.generateKey(readPartition)), nullValue()); - } - - // now we did acknowledge them in the reverse order. This way we check that only - // the highest value was acknowledged - for (Acknowledgment acknowledgment : FastList.newList(acknowledgments).reverseThis()) { - acknowledgment.acknowledge(); - } - - // now they are all acknowledged - for (Partition readPartition : readPartitions) { - assertThat(metadataStore.get(offsetManager.generateKey(readPartition)), equalTo(String.valueOf(20))); - } - } - -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/SingleBrokerWithPartitionSubsetTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/SingleBrokerWithPartitionSubsetTests.java deleted file mode 100644 index 456ef21a06..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/SingleBrokerWithPartitionSubsetTests.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package org.springframework.integration.kafka.listener; - -import org.junit.Ignore; -import org.junit.Rule; -import org.junit.Test; - -import org.springframework.integration.kafka.rule.KafkaEmbedded; - -/** - * @author Marius Bogoevici - */ -public class SingleBrokerWithPartitionSubsetTests extends AbstractMessageListenerContainerTests { - - @Rule - public final KafkaEmbedded kafkaEmbeddedBrokerRule = new KafkaEmbedded(1); - - @Override - public KafkaEmbedded getKafkaRule() { - return kafkaEmbeddedBrokerRule; - } - - @Test - public void testLowVolumeLowConcurrency() throws Exception { - createTopic(TEST_TOPIC, 4, 1, 1); - runMessageListenerTest(100, 2, 4, 100, 2, "none", TEST_TOPIC); - } - - @Test - @Ignore - public void testMediumVolumeLowConcurrency() throws Exception { - createTopic(TEST_TOPIC, 4, 1, 1); - runMessageListenerTest(100, 2, 4, 1000, 2, "none", TEST_TOPIC); - } - - @Test - @Ignore - public void testHighVolumeLowConcurrency() throws Exception { - createTopic(TEST_TOPIC, 4, 1, 1); - runMessageListenerTest(100, 2, 4, 10000, 2, "none", TEST_TOPIC); - } - - @Test - @Ignore - public void testLowVolumeMediumConcurrency() throws Exception { - createTopic(TEST_TOPIC, 10, 1, 1); - runMessageListenerTest(100, 5, 10, 100, 2, "none", TEST_TOPIC); - } - - @Test - @Ignore - public void testMediumVolumeMediumConcurrency() throws Exception { - createTopic(TEST_TOPIC, 10, 1, 1); - runMessageListenerTest(100, 5, 10, 1000, 2, "none", TEST_TOPIC); - } - - @Test - @Ignore - public void testHighVolumeMediumConcurrency() throws Exception { - createTopic(TEST_TOPIC, 10, 1, 1); - runMessageListenerTest(100, 5, 10, 100000, 2, "none", TEST_TOPIC); - } - - - @Test - @Ignore - public void testLowVolumeHighConcurrency() throws Exception { - createTopic(TEST_TOPIC, 100, 1, 1); - runMessageListenerTest(100, 20, 100, 1000, 2, "none", TEST_TOPIC); - } - - @Test - @Ignore - public void testMediumVolumeHighConcurrency() throws Exception { - createTopic(TEST_TOPIC, 100, 1, 1); - runMessageListenerTest(100, 20, 100, 10000, 2, "none", TEST_TOPIC); - } - - @Test - @Ignore - public void testHighVolumeHighConcurrency() throws Exception { - createTopic(TEST_TOPIC, 100, 1, 1); - runMessageListenerTest(100, 20, 100, 100000, 2, "none", TEST_TOPIC); - } - - -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/TestPartitioner.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/TestPartitioner.java deleted file mode 100644 index 46ff1f5e71..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/TestPartitioner.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package org.springframework.integration.kafka.listener; - -import kafka.producer.Partitioner; -import kafka.utils.VerifiableProperties; - -/** - * @author Marius Bogoevici - */ -public class TestPartitioner implements Partitioner { - - public TestPartitioner(VerifiableProperties properties) { - } - - @Override - public int partition(Object key, int numPartitions) { - if (key != null) { - if (key instanceof Number) { - return ((Number) key).intValue() % numPartitions; - } - else { - try { - return Integer.parseInt(key.toString()); - } - catch (NumberFormatException e) { - return 0; - } - } - } - else { - return 0; - } - } -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/WrongTopicNameTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/WrongTopicNameTests.java deleted file mode 100644 index b0ca1e59a0..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/WrongTopicNameTests.java +++ /dev/null @@ -1,90 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package org.springframework.integration.kafka.listener; - -import static org.hamcrest.collection.IsCollectionWithSize.hasSize; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.fail; -import static org.mockito.Mockito.mock; - -import org.junit.Rule; -import org.junit.Test; - -import org.springframework.beans.factory.BeanFactory; -import org.springframework.integration.kafka.core.ConnectionFactory; -import org.springframework.integration.kafka.core.TopicNotFoundException; -import org.springframework.integration.kafka.inbound.KafkaMessageDrivenChannelAdapter; -import org.springframework.integration.kafka.rule.KafkaEmbedded; -import org.springframework.integration.kafka.rule.KafkaRule; -import org.springframework.integration.kafka.serializer.common.StringDecoder; -import org.springframework.messaging.Message; -import org.springframework.messaging.MessageChannel; - -/** - * @author Marius Bogoevici - */ -public class WrongTopicNameTests extends AbstractMessageListenerContainerTests { - - @Rule - public final KafkaEmbedded kafkaEmbeddedBrokerRule = new KafkaEmbedded(1); - - @Override - public KafkaRule getKafkaRule() { - return kafkaEmbeddedBrokerRule; - } - - @Test(expected = TopicNotFoundException.class) - @SuppressWarnings("serial") - public void testWrongTopicNameFails() throws Exception { - createTopic(TEST_TOPIC, 5, 1, 1); - - ConnectionFactory connectionFactory = getKafkaBrokerConnectionFactory(); - - final KafkaMessageListenerContainer kafkaMessageListenerContainer - = new KafkaMessageListenerContainer(connectionFactory, "WRONG-TOPIC"); - kafkaMessageListenerContainer.setMaxFetch(100); - kafkaMessageListenerContainer.setConcurrency(2); - - KafkaMessageDrivenChannelAdapter kafkaMessageDrivenChannelAdapter = - new KafkaMessageDrivenChannelAdapter(kafkaMessageListenerContainer); - - StringDecoder decoder = new StringDecoder(); - kafkaMessageDrivenChannelAdapter.setKeyDecoder(decoder); - kafkaMessageDrivenChannelAdapter.setPayloadDecoder(decoder); - kafkaMessageDrivenChannelAdapter.setBeanFactory(mock(BeanFactory.class)); - kafkaMessageDrivenChannelAdapter.setOutputChannel(new MessageChannel() { - @Override - public boolean send(Message message) { - fail(); - return true; - } - - @Override - public boolean send(Message message, long timeout) { - fail(); - return true; - } - }); - - kafkaMessageDrivenChannelAdapter.afterPropertiesSet(); - kafkaMessageDrivenChannelAdapter.start(); - - - } - -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/ZookeeperConfigurationTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/ZookeeperConfigurationTests.java deleted file mode 100644 index 75e9b214d4..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/listener/ZookeeperConfigurationTests.java +++ /dev/null @@ -1,110 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package org.springframework.integration.kafka.listener; - -import static org.hamcrest.collection.IsCollectionWithSize.hasSize; -import static org.hamcrest.core.IsEqual.equalTo; -import static org.junit.Assert.assertThat; -import static org.springframework.integration.kafka.util.MessageUtils.decodeKey; -import static org.springframework.integration.kafka.util.MessageUtils.decodePayload; - -import java.util.ArrayList; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import com.gs.collections.api.multimap.list.MutableListMultimap; -import com.gs.collections.impl.multimap.list.SynchronizedPutFastListMultimap; -import kafka.serializer.StringDecoder; -import kafka.utils.VerifiableProperties; -import org.junit.Rule; -import org.junit.Test; - -import org.springframework.integration.kafka.core.Configuration; -import org.springframework.integration.kafka.core.ConnectionFactory; -import org.springframework.integration.kafka.core.KafkaMessage; -import org.springframework.integration.kafka.core.Partition; -import org.springframework.integration.kafka.core.ZookeeperConfiguration; -import org.springframework.integration.kafka.rule.KafkaEmbedded; -import org.springframework.integration.kafka.support.ZookeeperConnect; - -/** - * @author Marius Bogoevici - */ -public class ZookeeperConfigurationTests extends AbstractMessageListenerContainerTests { - - @Rule - public KafkaEmbedded kafkaEmbeddedBrokerRule = new KafkaEmbedded(2); - - @Override - public KafkaEmbedded getKafkaRule() { - return kafkaEmbeddedBrokerRule; - } - - @Override - public Configuration getKafkaConfiguration() { - ZookeeperConnect zookeeperConnect = new ZookeeperConnect(); - zookeeperConnect.setZkConnect(kafkaEmbeddedBrokerRule.getZookeeperConnectionString()); - return new ZookeeperConfiguration(zookeeperConnect); - } - - @Test - public void testLowVolumeLowConcurrency() throws Exception { - - int partitionCount = 5; - - createTopic(TEST_TOPIC, partitionCount, 2, 1); - - ConnectionFactory connectionFactory = getKafkaBrokerConnectionFactory(); - ArrayList readPartitions = new ArrayList(); - for (int i = 0; i < partitionCount; i++) { - readPartitions.add(new Partition(TEST_TOPIC, i)); - } - final KafkaMessageListenerContainer kafkaMessageListenerContainer = new KafkaMessageListenerContainer(connectionFactory, readPartitions.toArray(new Partition[readPartitions.size()])); - kafkaMessageListenerContainer.setMaxFetch(100); - kafkaMessageListenerContainer.setConcurrency(2); - - int expectedMessageCount = 100; - - final MutableListMultimap receivedData = new SynchronizedPutFastListMultimap(); - final CountDownLatch latch = new CountDownLatch(expectedMessageCount); - kafkaMessageListenerContainer.setMessageListener(new MessageListener() { - @Override - public void onMessage(KafkaMessage message) { - StringDecoder decoder = new StringDecoder(new VerifiableProperties()); - receivedData.put(message.getMetadata().getPartition().getId(),new KeyedMessageWithOffset(decodeKey(message, decoder), decodePayload(message, decoder), message.getMetadata().getOffset(), Thread.currentThread().getName(), message.getMetadata().getPartition().getId())); - latch.countDown(); - } - }); - - kafkaMessageListenerContainer.start(); - - createMessageSender("none").send(createMessages(100, TEST_TOPIC, partitionCount)); - - latch.await((expectedMessageCount / 5000) + 1, TimeUnit.MINUTES); - kafkaMessageListenerContainer.stop(); - - assertThat(receivedData.valuesView().toList(), hasSize(expectedMessageCount)); - assertThat(latch.getCount(), equalTo(0L)); - System.out.println("All messages received ... checking "); - - validateMessageReceipt(receivedData, 2, partitionCount, expectedMessageCount, 1); - - } - - -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/offset/AbstractOffsetManagerTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/offset/AbstractOffsetManagerTests.java deleted file mode 100644 index 1cdd3d94ba..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/offset/AbstractOffsetManagerTests.java +++ /dev/null @@ -1,302 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.offset; - -import static org.hamcrest.CoreMatchers.equalTo; -import static org.junit.Assert.assertThat; - -import java.util.Arrays; -import java.util.HashMap; -import java.util.Map; -import java.util.Properties; - -import kafka.api.OffsetRequest; -import org.apache.kafka.clients.producer.KafkaProducer; -import org.apache.kafka.clients.producer.Producer; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.junit.Rule; -import org.junit.Test; - -import org.springframework.integration.kafka.core.DefaultConnectionFactory; -import org.springframework.integration.kafka.core.Partition; -import org.springframework.integration.kafka.core.PartitionNotFoundException; -import org.springframework.integration.kafka.core.ZookeeperConfiguration; -import org.springframework.integration.kafka.listener.OffsetManager; -import org.springframework.integration.kafka.rule.KafkaEmbedded; -import org.springframework.integration.kafka.rule.KafkaRule; -import org.springframework.integration.kafka.serializer.common.StringEncoder; -import org.springframework.integration.kafka.util.EncoderAdaptingSerializer; -import org.springframework.integration.kafka.util.TopicUtils; - -/** - * @author Marius Bogoevici - */ -public abstract class AbstractOffsetManagerTests { - - @Rule - public KafkaRule kafkaRule = new KafkaEmbedded(1); - - private String TEST_TOPIC = "si_test_topic"; - - @Test(expected = PartitionNotFoundException.class) - public void testFailureWhenPartitionDoesNotExist() throws Exception { - OffsetManager offsetManager = createOffsetManager(OffsetRequest.EarliestTime(), "offset"); - offsetManager.getOffset(new Partition(TEST_TOPIC, 1)); - } - - @Test - public void testInitialization() throws Exception { - - int numPartitions = 3; - - TopicUtils.ensureTopicCreated(kafkaRule.getZookeeperConnectionString(), TEST_TOPIC, numPartitions, 1); - - OffsetManager offsetManager1 = createOffsetManager(OffsetRequest.EarliestTime(), "offset1"); - - Partition[] partitions = new Partition[numPartitions]; - for (int i = 0; i < numPartitions; i++) { - partitions[i] = new Partition(TEST_TOPIC, i); - } - // Earliest - assertThat(offsetManager1.getOffset(partitions[0]), equalTo(0L)); - assertThat(offsetManager1.getOffset(partitions[1]), equalTo(0L)); - assertThat(offsetManager1.getOffset(partitions[2]), equalTo(0L)); - - // send data to increase offsets on topics - Producer producer = createProducer(); - for (int i = 0; i < 10; i++) { - producer.send(new ProducerRecord(TEST_TOPIC, i%3, String.valueOf(i), String.valueOf(i))).get(); - } - - // earliest time resets at the start of the queue - OffsetManager offsetManager2 = createOffsetManager(OffsetRequest.EarliestTime(), "offset2"); - assertThat(offsetManager2.getOffset(partitions[0]), equalTo(0L)); - assertThat(offsetManager2.getOffset(partitions[1]), equalTo(0L)); - assertThat(offsetManager2.getOffset(partitions[2]), equalTo(0L)); - - offsetManager1.close(); - - // latest time resets at the end of the queue - OffsetManager offsetManager3 = createOffsetManager(OffsetRequest.LatestTime(), "offset3"); - assertThat(offsetManager3.getOffset(partitions[0]), equalTo(4L)); - assertThat(offsetManager3.getOffset(partitions[1]), equalTo(3L)); - assertThat(offsetManager3.getOffset(partitions[2]), equalTo(3L)); - - } - - @Test - public void testUpdate() throws Exception { - int numPartitions = 3; - - TopicUtils.ensureTopicCreated(kafkaRule.getZookeeperConnectionString(), TEST_TOPIC, numPartitions, 1); - - OffsetManager offsetManager1 = createOffsetManager(OffsetRequest.EarliestTime(), "offset1"); - - Partition[] partitions = new Partition[numPartitions]; - for (int i = 0; i < numPartitions; i++) { - partitions[i] = new Partition(TEST_TOPIC, i); - } - // Earliest - assertThat(offsetManager1.getOffset(partitions[0]), equalTo(0L)); - assertThat(offsetManager1.getOffset(partitions[1]), equalTo(0L)); - assertThat(offsetManager1.getOffset(partitions[2]), equalTo(0L)); - - // send data to increase offsets on topics - Producer producer = createProducer(); - for (int i = 0; i < 10; i++) { - producer.send(new ProducerRecord(TEST_TOPIC, - i%numPartitions, String.valueOf(i), String.valueOf(i))).get(); - } - - assertThat(offsetManager1.getOffset(partitions[0]), equalTo(0L)); - assertThat(offsetManager1.getOffset(partitions[1]), equalTo(0L)); - assertThat(offsetManager1.getOffset(partitions[2]), equalTo(0L)); - - offsetManager1.updateOffset(partitions[0], 5L); - offsetManager1.updateOffset(partitions[1], 4L); - offsetManager1.updateOffset(partitions[2], 3L); - assertThat(offsetManager1.getOffset(partitions[0]), equalTo(5L)); - assertThat(offsetManager1.getOffset(partitions[1]), equalTo(4L)); - assertThat(offsetManager1.getOffset(partitions[2]), equalTo(3L)); - - - // a new offset manager with the same consumerId will observe the updates - OffsetManager newOffsetManager1 = createOffsetManager(OffsetRequest.EarliestTime(), "offset1"); - assertThat(newOffsetManager1.getOffset(partitions[0]), equalTo(5L)); - assertThat(newOffsetManager1.getOffset(partitions[1]), equalTo(4L)); - assertThat(newOffsetManager1.getOffset(partitions[2]), equalTo(3L)); - - offsetManager1.close(); - - // a new offset manager with a different consumerId will not observe the updates - OffsetManager offsetManager2 = createOffsetManager(OffsetRequest.EarliestTime(), "offset2"); - assertThat(offsetManager2.getOffset(partitions[0]), equalTo(0L)); - assertThat(offsetManager2.getOffset(partitions[1]), equalTo(0L)); - assertThat(offsetManager2.getOffset(partitions[2]), equalTo(0L)); - - } - - @Test - public void testRemove() throws Exception { - int numPartitions = 3; - - TopicUtils.ensureTopicCreated(kafkaRule.getZookeeperConnectionString(), TEST_TOPIC, numPartitions, 1); - - OffsetManager offsetManager1 = createOffsetManager(OffsetRequest.EarliestTime(), "offset1"); - - Partition[] partitions = new Partition[numPartitions]; - for (int i = 0; i < numPartitions; i++) { - partitions[i] = new Partition(TEST_TOPIC, i); - } - // Earliest - assertThat(offsetManager1.getOffset(partitions[0]), equalTo(0L)); - assertThat(offsetManager1.getOffset(partitions[1]), equalTo(0L)); - assertThat(offsetManager1.getOffset(partitions[2]), equalTo(0L)); - - // send data to increase offsets on topics - Producer producer = createProducer(); - for (int i = 0; i < 6; i++) { - producer.send(new ProducerRecord(TEST_TOPIC, i%numPartitions, String.valueOf(i), String.valueOf(i))).get(); - } - - assertThat(offsetManager1.getOffset(partitions[0]), equalTo(0L)); - assertThat(offsetManager1.getOffset(partitions[1]), equalTo(0L)); - assertThat(offsetManager1.getOffset(partitions[2]), equalTo(0L)); - - offsetManager1.updateOffset(partitions[0], 5L); - offsetManager1.updateOffset(partitions[1], 4L); - offsetManager1.updateOffset(partitions[2], 3L); - - offsetManager1.deleteOffset(partitions[1]); - offsetManager1.deleteOffset(partitions[2]); - - assertThat(offsetManager1.getOffset(partitions[0]), equalTo(5L)); - // partition 1,2 will be reset - assertThat(offsetManager1.getOffset(partitions[1]), equalTo(0L)); - assertThat(offsetManager1.getOffset(partitions[2]), equalTo(0L)); - - // a new offset manager with the same consumerId will observe the updates - OffsetManager newOffsetManager1 = createOffsetManager(OffsetRequest.EarliestTime(), "offset1"); - assertThat(newOffsetManager1.getOffset(partitions[0]), equalTo(5L)); - assertThat(newOffsetManager1.getOffset(partitions[1]), equalTo(0L)); - assertThat(newOffsetManager1.getOffset(partitions[2]), equalTo(0L)); - - offsetManager1.close(); - - // a new offset manager with a different consumerId will not observe the updates - OffsetManager offsetManager2 = createOffsetManager(OffsetRequest.LatestTime(), "offset2"); - assertThat(offsetManager2.getOffset(partitions[0]), equalTo(2L)); - assertThat(offsetManager2.getOffset(partitions[1]), equalTo(2L)); - assertThat(offsetManager2.getOffset(partitions[2]), equalTo(2L)); - } - - @Test - public void testInitialOffsets() throws Exception { - int numPartitions = 3; - - TopicUtils.ensureTopicCreated(kafkaRule.getZookeeperConnectionString(), TEST_TOPIC, numPartitions, 1); - - Partition[] partitions = new Partition[numPartitions]; - for (int i = 0; i < numPartitions; i++) { - partitions[i] = new Partition(TEST_TOPIC, i); - } - - HashMap initialOffsets = new HashMap(); - initialOffsets.put(partitions[0], 8L); - initialOffsets.put(partitions[1], 9L); - initialOffsets.put(partitions[2], 10L); - - - - OffsetManager offsetManager1 = createOffsetManager(OffsetRequest.EarliestTime(), "offset1", initialOffsets); - - // send data to increase offsets on topics - Producer producer = createProducer(); - for (int i = 0; i < 6; i++) { - producer.send(new ProducerRecord(TEST_TOPIC, i%numPartitions, String.valueOf(i), String.valueOf(i))).get(); - } - - // The offset manager starts at the configured offsets - assertThat(offsetManager1.getOffset(partitions[0]), equalTo(8L)); - assertThat(offsetManager1.getOffset(partitions[1]), equalTo(9L)); - assertThat(offsetManager1.getOffset(partitions[2]), equalTo(10L)); - - // a new offset manager with the same consumerId will not observe any updates yet - OffsetManager newOffsetManager1 = createOffsetManager(OffsetRequest.EarliestTime(), "offset1"); - assertThat(newOffsetManager1.getOffset(partitions[0]), equalTo(0L)); - assertThat(newOffsetManager1.getOffset(partitions[1]), equalTo(0L)); - assertThat(newOffsetManager1.getOffset(partitions[2]), equalTo(0L)); - - offsetManager1.updateOffset(partitions[0], 9L); - offsetManager1.updateOffset(partitions[1], 10L); - offsetManager1.updateOffset(partitions[2], 11L); - - assertThat(offsetManager1.getOffset(partitions[0]), equalTo(9L)); - assertThat(offsetManager1.getOffset(partitions[1]), equalTo(10L)); - assertThat(offsetManager1.getOffset(partitions[2]), equalTo(11L)); - - // a new offset manager with the same consumerId will observe the updates - OffsetManager newOffsetManager2 = createOffsetManager(OffsetRequest.EarliestTime(), "offset1"); - assertThat(newOffsetManager2.getOffset(partitions[0]), equalTo(9L)); - assertThat(newOffsetManager2.getOffset(partitions[1]), equalTo(10L)); - assertThat(newOffsetManager2.getOffset(partitions[2]), equalTo(11L)); - - offsetManager1.deleteOffset(partitions[0]); - offsetManager1.deleteOffset(partitions[1]); - - // after delete, the offset manager starts in an initial position - assertThat(offsetManager1.getOffset(partitions[0]), equalTo(8L)); - assertThat(offsetManager1.getOffset(partitions[1]), equalTo(9L)); - assertThat(offsetManager1.getOffset(partitions[2]), equalTo(11L)); - - offsetManager1.resetOffsets(Arrays.asList(partitions)); - // after reset, the offset manager starts at the reference timestamp - assertThat(offsetManager1.getOffset(partitions[0]), equalTo(0L)); - assertThat(offsetManager1.getOffset(partitions[1]), equalTo(0L)); - assertThat(offsetManager1.getOffset(partitions[2]), equalTo(0L)); - - offsetManager1.close(); - - // a new offset manager with a different consumerId will not observe the updates - OffsetManager offsetManager2 = createOffsetManager(OffsetRequest.LatestTime(), "offset2"); - assertThat(offsetManager2.getOffset(partitions[0]), equalTo(2L)); - assertThat(offsetManager2.getOffset(partitions[1]), equalTo(2L)); - assertThat(offsetManager2.getOffset(partitions[2]), equalTo(2L)); - } - - protected DefaultConnectionFactory createConnectionFactory() throws Exception { - DefaultConnectionFactory connectionFactory - = new DefaultConnectionFactory(new ZookeeperConfiguration(kafkaRule.getZookeeperConnectionString())); - connectionFactory.afterPropertiesSet(); - return connectionFactory; - } - - protected Producer createProducer() { - Properties properties = new Properties(); - properties.setProperty("bootstrap.servers", kafkaRule.getBrokersAsString()); - EncoderAdaptingSerializer serializer = new EncoderAdaptingSerializer<>(new StringEncoder()); - return new KafkaProducer(properties, serializer, serializer); - } - - protected OffsetManager createOffsetManager(long referenceTimestamp, String consumerId) throws Exception { - return createOffsetManager(referenceTimestamp, consumerId, new HashMap()); - } - - protected abstract OffsetManager createOffsetManager(long referenceTimestamp, - String consumerId, Map initialOffsets) throws Exception; - -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/offset/CodecTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/offset/CodecTests.java deleted file mode 100644 index d2ee323a46..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/offset/CodecTests.java +++ /dev/null @@ -1,52 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.offset; - -import static org.hamcrest.CoreMatchers.equalTo; -import static org.junit.Assert.assertThat; - -import org.junit.Test; - -import org.springframework.integration.kafka.core.Partition; -import org.springframework.integration.kafka.listener.KafkaTopicOffsetManager; - -/** - * @author Marius Bogoevici - */ -public class CodecTests { - - public static final String SOME_CONSUMER = "someConsumer"; - - public static final String SOME_TOPIC = "someTopic"; - - public static final int SOME_PARTITION_ID = 42; - - @Test - public void testKeyCodec() { - KafkaTopicOffsetManager.KeySerializerDecoder codec = new KafkaTopicOffsetManager.KeySerializerDecoder(); - - byte[] encodedValue = codec.serialize("#unused", new KafkaTopicOffsetManager.Key(SOME_CONSUMER, - new Partition(SOME_TOPIC, SOME_PARTITION_ID))); - - KafkaTopicOffsetManager.Key key = codec.fromBytes(encodedValue); - - assertThat(key.getConsumerId(), equalTo(SOME_CONSUMER)); - assertThat(key.getPartition().getTopic(), equalTo(SOME_TOPIC)); - assertThat(key.getPartition().getId(), equalTo(SOME_PARTITION_ID)); - } - -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/offset/KafkaNativeOffsetManagerTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/offset/KafkaNativeOffsetManagerTests.java deleted file mode 100644 index f37ba4222a..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/offset/KafkaNativeOffsetManagerTests.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.offset; - -import java.util.Map; - -import org.springframework.integration.kafka.core.DefaultConnectionFactory; -import org.springframework.integration.kafka.core.Partition; -import org.springframework.integration.kafka.core.ZookeeperConfiguration; -import org.springframework.integration.kafka.listener.KafkaNativeOffsetManager; -import org.springframework.integration.kafka.listener.OffsetManager; -import org.springframework.integration.kafka.support.ZookeeperConnect; - -/** - * @author Chris Lemper - * @since 1.3 - */ -public class KafkaNativeOffsetManagerTests extends AbstractOffsetManagerTests { - - @Override - protected OffsetManager createOffsetManager(long referenceTimestamp, String consumerId, - Map initialOffsets) throws Exception { - ZookeeperConnect zookeeperConnect = new ZookeeperConnect(kafkaRule.getZookeeperConnectionString()); - KafkaNativeOffsetManager kafkaNativeOffsetManager = - new KafkaNativeOffsetManager(new DefaultConnectionFactory(new ZookeeperConfiguration(zookeeperConnect)), - zookeeperConnect, - initialOffsets); - kafkaNativeOffsetManager.setConsumerId(consumerId); - kafkaNativeOffsetManager.afterPropertiesSet(); - kafkaNativeOffsetManager.setReferenceTimestamp(referenceTimestamp); - return kafkaNativeOffsetManager; - } - -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/offset/KafkaTopicOffsetManagerTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/offset/KafkaTopicOffsetManagerTests.java deleted file mode 100644 index 17ed99a4ae..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/offset/KafkaTopicOffsetManagerTests.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.offset; - -import java.util.Map; - -import org.springframework.integration.kafka.core.Partition; -import org.springframework.integration.kafka.listener.KafkaTopicOffsetManager; -import org.springframework.integration.kafka.listener.OffsetManager; -import org.springframework.integration.kafka.support.ZookeeperConnect; - -/** - * @author Marius Bogoevici - */ -public class KafkaTopicOffsetManagerTests extends AbstractOffsetManagerTests { - - @Override - protected OffsetManager createOffsetManager(long referenceTimestamp, String consumerId, - Map initialOffsets) throws Exception { - KafkaTopicOffsetManager kafkaTopicOffsetManager = - new KafkaTopicOffsetManager(new ZookeeperConnect(kafkaRule.getZookeeperConnectionString()), - "offsets-spring-test", - initialOffsets); - kafkaTopicOffsetManager.setConsumerId(consumerId); - kafkaTopicOffsetManager.afterPropertiesSet(); - kafkaTopicOffsetManager.setReferenceTimestamp(referenceTimestamp); - return kafkaTopicOffsetManager; - } - -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/offset/MetadataStoreOffsetManagerTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/offset/MetadataStoreOffsetManagerTests.java deleted file mode 100644 index 0ceae0f0ae..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/offset/MetadataStoreOffsetManagerTests.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.offset; - -import java.util.Map; - -import org.junit.Before; - -import org.springframework.integration.kafka.core.Partition; -import org.springframework.integration.kafka.listener.MetadataStoreOffsetManager; -import org.springframework.integration.kafka.listener.OffsetManager; -import org.springframework.integration.metadata.SimpleMetadataStore; - -/** - * @author Marius Bogoevici - */ -public class MetadataStoreOffsetManagerTests extends AbstractOffsetManagerTests { - - - // tests share a MetadataStore instance internally, simulating the behaviour of persistent stores - private SimpleMetadataStore metadataStore; - - @Before - public void setUp() { - metadataStore = new SimpleMetadataStore(); - } - - @Override - protected OffsetManager createOffsetManager(long referenceTimestamp, String consumerId, - Map initialOffsets) throws Exception { - MetadataStoreOffsetManager metadataStoreOffsetManager; - metadataStoreOffsetManager = new MetadataStoreOffsetManager(createConnectionFactory(), initialOffsets); - metadataStoreOffsetManager.setMetadataStore(this.metadataStore); - metadataStoreOffsetManager.setReferenceTimestamp(referenceTimestamp); - metadataStoreOffsetManager.setConsumerId(consumerId); - return metadataStoreOffsetManager; - } - -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/offset/WindowingOffsetManagerTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/offset/WindowingOffsetManagerTests.java deleted file mode 100644 index a7e25f86a7..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/offset/WindowingOffsetManagerTests.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Copyright 2016 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.offset; - -import java.util.Map; - -import org.springframework.integration.kafka.core.DefaultConnectionFactory; -import org.springframework.integration.kafka.core.Partition; -import org.springframework.integration.kafka.core.ZookeeperConfiguration; -import org.springframework.integration.kafka.listener.KafkaNativeOffsetManager; -import org.springframework.integration.kafka.listener.OffsetManager; -import org.springframework.integration.kafka.listener.WindowingOffsetManager; -import org.springframework.integration.kafka.support.ZookeeperConnect; - -/** - * @author Artem Bilan - * @since 1.3.1 - */ -public class WindowingOffsetManagerTests extends AbstractOffsetManagerTests { - - @Override - protected OffsetManager createOffsetManager(long referenceTimestamp, - String consumerId, - Map initialOffsets) throws Exception { - ZookeeperConnect zookeeperConnect = new ZookeeperConnect(kafkaRule.getZookeeperConnectionString()); - KafkaNativeOffsetManager kafkaNativeOffsetManager = - new KafkaNativeOffsetManager(new DefaultConnectionFactory(new ZookeeperConfiguration(zookeeperConnect)), - zookeeperConnect, - initialOffsets); - kafkaNativeOffsetManager.setConsumerId(consumerId); - kafkaNativeOffsetManager.afterPropertiesSet(); - kafkaNativeOffsetManager.setReferenceTimestamp(referenceTimestamp); - - WindowingOffsetManager windowingOffsetManager = new WindowingOffsetManager(kafkaNativeOffsetManager); - windowingOffsetManager.setCount(2); - windowingOffsetManager.setTimespan(10); - windowingOffsetManager.afterPropertiesSet(); - return windowingOffsetManager; - } -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/outbound/OutboundTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/outbound/OutboundTests.java deleted file mode 100644 index 70c40d92ea..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/outbound/OutboundTests.java +++ /dev/null @@ -1,465 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.outbound; - -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.UUID; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; - -import com.gs.collections.api.multimap.MutableMultimap; -import com.gs.collections.impl.factory.Multimaps; -import kafka.admin.AdminUtils; -import kafka.api.OffsetRequest; -import kafka.common.TopicExistsException; -import kafka.serializer.Decoder; -import kafka.serializer.Encoder; -import org.apache.kafka.clients.producer.RecordMetadata; -import org.apache.kafka.common.serialization.ByteArraySerializer; -import org.apache.kafka.common.serialization.StringSerializer; -import org.junit.After; -import org.junit.Rule; -import org.junit.Test; - -import org.springframework.beans.factory.BeanFactory; -import org.springframework.expression.spel.standard.SpelExpressionParser; -import org.springframework.integration.kafka.core.DefaultConnectionFactory; -import org.springframework.integration.kafka.core.KafkaMessage; -import org.springframework.integration.kafka.core.ZookeeperConfiguration; -import org.springframework.integration.kafka.listener.KafkaMessageListenerContainer; -import org.springframework.integration.kafka.listener.MessageListener; -import org.springframework.integration.kafka.listener.MetadataStoreOffsetManager; -import org.springframework.integration.kafka.rule.KafkaEmbedded; -import org.springframework.integration.kafka.rule.KafkaRule; -import org.springframework.integration.kafka.serializer.common.StringDecoder; -import org.springframework.integration.kafka.serializer.common.StringEncoder; -import org.springframework.integration.kafka.support.KafkaHeaders; -import org.springframework.integration.kafka.support.KafkaProducerContext; -import org.springframework.integration.kafka.support.ProducerConfiguration; -import org.springframework.integration.kafka.support.ProducerFactoryBean; -import org.springframework.integration.kafka.support.ProducerListener; -import org.springframework.integration.kafka.support.ProducerMetadata; -import org.springframework.integration.kafka.support.ZookeeperConnect; -import org.springframework.integration.kafka.util.EncoderAdaptingSerializer; -import org.springframework.integration.kafka.util.MessageUtils; -import org.springframework.integration.kafka.util.TopicUtils; -import org.springframework.messaging.support.MessageBuilder; - -import static org.hamcrest.Matchers.contains; -import static org.hamcrest.Matchers.endsWith; -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.hasItem; -import static org.hamcrest.Matchers.hasSize; -import static org.junit.Assert.assertThat; -import static org.junit.Assert.assertTrue; -import static org.mockito.Mockito.mock; - -/** - * @author Gary Russell - * @author Marius Bogoevici - * @since 1.0 - */ -public class OutboundTests { - - private static final String TOPIC = "springintegrationtest"; - - private static final String TOPIC2 = "springintegrationtest2"; - - @Rule - public KafkaRule kafkaRule = new KafkaEmbedded(1); - - private final Decoder decoder = new StringDecoder(); - - @After - public void tearDown() { - try { - AdminUtils.deleteTopic(kafkaRule.getZkClient(), TOPIC); - AdminUtils.deleteTopic(kafkaRule.getZkClient(), TOPIC2); - } - catch (Exception e) { - } - } - - @Test - public void testAsyncProducerFlushed() throws Exception { - - // create the topic - - try { - TopicUtils.ensureTopicCreated(kafkaRule.getZookeeperConnectionString(), TOPIC, 1, 1); - } - catch (TopicExistsException e) { - // do nothing - } - - final String suffix = UUID.randomUUID().toString(); - - KafkaMessageListenerContainer kafkaMessageListenerContainer = createMessageListenerContainer(TOPIC); - - int expectedMessageCount = 2; - final List payloads = new ArrayList(); - final CountDownLatch latch = new CountDownLatch(expectedMessageCount); - kafkaMessageListenerContainer.setMessageListener(new MessageListener() { - - @Override - public void onMessage(KafkaMessage message) { - payloads.add(MessageUtils.decodePayload(message, decoder)); - latch.countDown(); - } - - }); - - kafkaMessageListenerContainer.start(); - - KafkaProducerContext producerContext = createProducerContext(); - KafkaProducerMessageHandler handler = - new KafkaProducerMessageHandler(producerContext); - - handler.handleMessage(MessageBuilder.withPayload(("foo" + suffix).getBytes()) - .setHeader(KafkaHeaders.MESSAGE_KEY, "3") - .setHeader(KafkaHeaders.TOPIC, TOPIC) - .build()); - - SpelExpressionParser parser = new SpelExpressionParser(); - handler.setMessageKeyExpression(parser.parseExpression("headers.foo")); - handler.setTopicExpression(parser.parseExpression("headers.bar")); - handler.setBeanFactory(mock(BeanFactory.class)); - handler.afterPropertiesSet(); - - handler.handleMessage(MessageBuilder.withPayload("bar" + suffix) - .setHeader("foo", "3") - .setHeader("bar", TOPIC) - .build()); - - producerContext.stop(); - - latch.await(1000, TimeUnit.MILLISECONDS); - assertThat(latch.getCount(), equalTo(0L)); - for (String payload : payloads) { - assertThat(payload, endsWith(suffix)); - } - kafkaMessageListenerContainer.stop(); - } - - @Test - public void testHeaderRoutingAndAsyncCallback() throws Exception { - - // create the topic - - try { - TopicUtils.ensureTopicCreated(kafkaRule.getZookeeperConnectionString(), TOPIC, 1, 1); - } - catch (TopicExistsException e) { - // do nothing - } - - try { - TopicUtils.ensureTopicCreated(kafkaRule.getZookeeperConnectionString(), TOPIC2, 1, 1); - } - catch (TopicExistsException e) { - // do nothing - } - - final String suffix = UUID.randomUUID().toString(); - - KafkaMessageListenerContainer kafkaMessageListenerContainer = createMessageListenerContainer(TOPIC, TOPIC2); - - final Decoder decoder = new StringDecoder(); - - int expectedMessageCount = 4; - final MutableMultimap payloadsByTopic = Multimaps.mutable.list.with(); - final CountDownLatch latch = new CountDownLatch(expectedMessageCount); - kafkaMessageListenerContainer.setMessageListener(new MessageListener() { - - @Override - public void onMessage(KafkaMessage message) { - payloadsByTopic.put(message.getMetadata().getPartition().getTopic(), - MessageUtils.decodePayload(message, decoder)); - latch.countDown(); - } - - }); - - kafkaMessageListenerContainer.start(); - - int expectedDeliveryConfirmations = 4; - final List results = new ArrayList(); - final CountDownLatch sendResultLatch = new CountDownLatch(expectedDeliveryConfirmations); - ProducerListener listener = new ProducerListener() { - - @Override - public void onSuccess(String topic, Integer partition, Object key, Object value, RecordMetadata recordMetadata) { - results.add(recordMetadata); - sendResultLatch.countDown(); - } - - @Override - public void onError(String topic, Integer partition, Object key, Object value, Exception exception) { - sendResultLatch.countDown(); - } - }; - - KafkaProducerContext producerContext = createProducerContext(listener); - KafkaProducerMessageHandler handler - = new KafkaProducerMessageHandler(producerContext); - - handler.handleMessage(MessageBuilder.withPayload("fooTopic1" + suffix) - .setHeader(KafkaHeaders.MESSAGE_KEY, "3") - .setHeader(KafkaHeaders.TOPIC, TOPIC) - .build()); - - handler.handleMessage(MessageBuilder.withPayload("fooTopic2" + suffix) - .setHeader(KafkaHeaders.MESSAGE_KEY, "3") - .setHeader(KafkaHeaders.TOPIC, TOPIC2) - .build()); - - SpelExpressionParser parser = new SpelExpressionParser(); - handler.setMessageKeyExpression(parser.parseExpression("headers.foo")); - handler.setTopicExpression(parser.parseExpression("headers.bar")); - handler.setBeanFactory(mock(BeanFactory.class)); - handler.afterPropertiesSet(); - - handler.handleMessage(MessageBuilder.withPayload("bar1" + suffix) - .setHeader("foo", "3") - .setHeader("bar", TOPIC) - .build()); - - handler.handleMessage(MessageBuilder.withPayload("bar2" + suffix) - .setHeader("foo", "3") - .setHeader("bar", TOPIC2) - .build()); - - assertTrue(sendResultLatch.await(10, TimeUnit.SECONDS)); - assertThat(results.size(), equalTo(expectedDeliveryConfirmations)); - - producerContext.stop(); - - latch.await(10000, TimeUnit.MILLISECONDS); - assertThat(latch.getCount(), equalTo(0L)); - // messages are routed to both topics - assertThat(payloadsByTopic.keysView(), hasItem(TOPIC)); - assertThat(payloadsByTopic.keysView(), hasItem(TOPIC2)); - assertThat(payloadsByTopic.toMap().get(TOPIC), contains("fooTopic1" + suffix, "bar1" + suffix)); - assertThat(payloadsByTopic.toMap().get(TOPIC2), contains("fooTopic2" + suffix, "bar2" + suffix)); - - kafkaMessageListenerContainer.stop(); - } - - @Test - public void testHeaderRoutingDisabled() throws Exception { - - // create the topic - - try { - TopicUtils.ensureTopicCreated(kafkaRule.getZookeeperConnectionString(), TOPIC, 1, 1); - } - catch (TopicExistsException e) { - // do nothing - } - - try { - TopicUtils.ensureTopicCreated(kafkaRule.getZookeeperConnectionString(), TOPIC2, 1, 1); - } - catch (TopicExistsException e) { - // do nothing - } - - final String suffix = UUID.randomUUID().toString(); - - KafkaMessageListenerContainer kafkaMessageListenerContainer = createMessageListenerContainer(TOPIC, TOPIC2); - - final Decoder decoder = new StringDecoder(); - - int expectedMessageCount = 2; - final MutableMultimap payloadsByTopic = Multimaps.mutable.list.with(); - final CountDownLatch latch = new CountDownLatch(expectedMessageCount); - kafkaMessageListenerContainer.setMessageListener(new MessageListener() { - - @Override - public void onMessage(KafkaMessage message) { - payloadsByTopic.put(message.getMetadata().getPartition().getTopic(), - MessageUtils.decodePayload(message, decoder)); - latch.countDown(); - } - - }); - - kafkaMessageListenerContainer.start(); - - int expectedDeliveryConfirmations = 2; - final List results = new ArrayList(); - final CountDownLatch sendResultLatch = new CountDownLatch(expectedDeliveryConfirmations); - ProducerListener listener = new ProducerListener() { - - @Override - public void onSuccess(String topic, Integer partition, Object key, Object value, RecordMetadata recordMetadata) { - results.add(recordMetadata); - sendResultLatch.countDown(); - } - - @Override - public void onError(String topic, Integer partition, Object key, Object value, Exception exception) { - sendResultLatch.countDown(); - } - }; - - KafkaProducerContext producerContext = createProducerContext(listener); - - KafkaProducerMessageHandler handler - = new KafkaProducerMessageHandler(producerContext); - - handler.setEnableHeaderRouting(false); - - handler.handleMessage(MessageBuilder.withPayload("fooTopic1Header" + suffix) - .setHeader(KafkaHeaders.MESSAGE_KEY, "3") - .setHeader(KafkaHeaders.TOPIC, TOPIC) - .build()); - - // even if the header is set to TOPIC2, it should be ignored - handler.handleMessage(MessageBuilder.withPayload("fooTopic2Header" + suffix) - .setHeader(KafkaHeaders.MESSAGE_KEY, "3") - .setHeader(KafkaHeaders.TOPIC, TOPIC2) - .build()); - - assertTrue(sendResultLatch.await(10, TimeUnit.SECONDS)); - assertThat(results.size(), equalTo(expectedDeliveryConfirmations)); - - producerContext.stop(); - - latch.await(10000, TimeUnit.MILLISECONDS); - assertThat(latch.getCount(), equalTo(0L)); - // messages are routed to both topics - assertThat(payloadsByTopic.keysView().size(), equalTo(1)); - assertThat(payloadsByTopic.keysView(), hasItem(TOPIC)); - assertThat(payloadsByTopic.toMap().get(TOPIC), - contains("fooTopic1Header" + suffix, "fooTopic2Header" + suffix)); - kafkaMessageListenerContainer.stop(); - } - - - @Test - public void testNoHeader() throws Exception { - - // create the topic - - try { - TopicUtils.ensureTopicCreated(kafkaRule.getZookeeperConnectionString(), - TOPIC, 1, 1); - } - catch (TopicExistsException e) { - // do nothing - } - - final String suffix = UUID.randomUUID().toString(); - - KafkaMessageListenerContainer kafkaMessageListenerContainer = createMessageListenerContainer(TOPIC); - - final Decoder decoder = new StringDecoder(); - - int expectedMessageCount = 1; - final MutableMultimap payloadsByTopic = Multimaps.mutable.list.with(); - final CountDownLatch latch = new CountDownLatch(expectedMessageCount); - kafkaMessageListenerContainer.setMessageListener(new MessageListener() { - - @Override - public void onMessage(KafkaMessage message) { - payloadsByTopic.put(message.getMetadata().getPartition().getTopic(), - MessageUtils.decodePayload(message, decoder)); - latch.countDown(); - } - - }); - - kafkaMessageListenerContainer.start(); - - KafkaProducerContext kafkaProducerContext = new KafkaProducerContext(); - ProducerMetadata producerMetadata = - new ProducerMetadata<>(TOPIC, String.class, byte[].class, - new StringSerializer(), new ByteArraySerializer()); - Properties props = new Properties(); - ProducerFactoryBean producer = - new ProducerFactoryBean<>(producerMetadata, kafkaRule.getBrokersAsString(), props); - ProducerConfiguration config = - new ProducerConfiguration<>(producerMetadata, producer.getObject()); - - Map> producerConfigurationMap = - Collections.>singletonMap(TOPIC, config); - kafkaProducerContext.setProducerConfigurations(producerConfigurationMap); - - KafkaProducerMessageHandler handler = new KafkaProducerMessageHandler(kafkaProducerContext); - - handler.setBeanFactory(mock(BeanFactory.class)); - handler.afterPropertiesSet(); - - handler.handleMessage(MessageBuilder.withPayload("fooTopic1" + suffix).build()); - - kafkaProducerContext.stop(); - - latch.await(1000, TimeUnit.MILLISECONDS); - assertThat(latch.getCount(), equalTo(0L)); - assertThat(payloadsByTopic.keysView(), hasItem(TOPIC)); - assertThat(payloadsByTopic.toMap().get(TOPIC).toList(), hasSize(1)); - assertThat(payloadsByTopic.toMap().get(TOPIC), contains("fooTopic1" + suffix)); - kafkaMessageListenerContainer.stop(); - } - - private KafkaMessageListenerContainer createMessageListenerContainer(String... topics) throws Exception { - ZookeeperConfiguration configuration = - new ZookeeperConfiguration(new ZookeeperConnect(kafkaRule.getZookeeperConnectionString())); - DefaultConnectionFactory connectionFactory = new DefaultConnectionFactory(configuration); - connectionFactory.afterPropertiesSet(); - final KafkaMessageListenerContainer kafkaMessageListenerContainer = - new KafkaMessageListenerContainer(connectionFactory, topics); - kafkaMessageListenerContainer.setMaxFetch(100); - kafkaMessageListenerContainer.setConcurrency(1); - MetadataStoreOffsetManager offsetManager = new MetadataStoreOffsetManager(connectionFactory); - // start reading at the end of the - offsetManager.setReferenceTimestamp(OffsetRequest.LatestTime()); - kafkaMessageListenerContainer.setOffsetManager(offsetManager); - return kafkaMessageListenerContainer; - } - - private KafkaProducerContext createProducerContext() throws Exception { - return createProducerContext(null); - } - - private KafkaProducerContext createProducerContext(ProducerListener producerListener) throws Exception { - KafkaProducerContext kafkaProducerContext = new KafkaProducerContext(); - Encoder encoder = new StringEncoder(); - ProducerMetadata producerMetadata = - new ProducerMetadata(TOPIC, String.class, String.class, - new EncoderAdaptingSerializer<>(encoder), new EncoderAdaptingSerializer<>(encoder)); - Properties props = new Properties(); - if (producerListener == null) { - props.put("linger.ms", "15000"); - } - ProducerFactoryBean producer = - new ProducerFactoryBean<>(producerMetadata, kafkaRule.getBrokersAsString(), props); - ProducerConfiguration config = - new ProducerConfiguration<>(producerMetadata, producer.getObject()); - config.setProducerListener(producerListener); - Map> producerConfigurationMap = - Collections.>singletonMap(TOPIC, config); - kafkaProducerContext.setProducerConfigurations(producerConfigurationMap); - return kafkaProducerContext; - } - -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/outbound/ProducerListenerTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/outbound/ProducerListenerTests.java deleted file mode 100644 index 5d43e61274..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/outbound/ProducerListenerTests.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.outbound; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertSame; -import static org.junit.Assert.assertThat; -import static org.mockito.Matchers.any; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.verifyNoMoreInteractions; - -import java.util.Collections; -import java.util.Map; - -import org.apache.kafka.clients.producer.Callback; -import org.apache.kafka.clients.producer.Producer; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.common.serialization.Serializer; -import org.hamcrest.CoreMatchers; -import org.junit.Test; -import org.mockito.ArgumentCaptor; -import org.mockito.Mockito; - -import org.springframework.beans.DirectFieldAccessor; -import org.springframework.integration.kafka.support.KafkaHeaders; -import org.springframework.integration.kafka.support.KafkaProducerContext; -import org.springframework.integration.kafka.support.ProducerConfiguration; -import org.springframework.integration.kafka.support.ProducerListener; -import org.springframework.integration.kafka.support.ProducerListenerInvokingCallback; -import org.springframework.integration.kafka.support.ProducerMetadata; -import org.springframework.messaging.support.MessageBuilder; - -/** - * @author Marius Bogoevici - */ -public class ProducerListenerTests { - - @Test - @SuppressWarnings({"unchecked","rawtypes"}) - public void testProducerListenerSet() throws Exception { - KafkaProducerContext producerContext = new KafkaProducerContext(); - Serializer serializer = Mockito.mock(Serializer.class); - ProducerMetadata producerMetadata - = new ProducerMetadata<>("default", Object.class, Object.class, serializer, serializer); - Producer producer = Mockito.mock(Producer.class); - ProducerConfiguration producerConfiguration - = new ProducerConfiguration<>(producerMetadata, producer); - ProducerListener producerListener = mock(ProducerListener.class); - producerConfiguration.setProducerListener(producerListener); - Map> producerConfigurations - = Collections.>singletonMap("default", producerConfiguration); - producerContext.setProducerConfigurations(producerConfigurations); - KafkaProducerMessageHandler handler = new KafkaProducerMessageHandler(producerContext); - handler.handleMessage( - MessageBuilder.withPayload("somePayload") - .setHeader(KafkaHeaders.PARTITION_ID, 2) - .setHeader(KafkaHeaders.MESSAGE_KEY, "someKey") - .build()); - final ArgumentCaptor argument = ArgumentCaptor.forClass(Callback.class); - verify(producer).send(any(ProducerRecord.class), argument.capture()); - Callback callback = argument.getValue(); - assertThat(callback, CoreMatchers.instanceOf(ProducerListenerInvokingCallback.class)); - DirectFieldAccessor fieldAccessor = new DirectFieldAccessor(callback); - assertEquals(fieldAccessor.getPropertyValue("topic"),"default"); - assertEquals(fieldAccessor.getPropertyValue("partition"),2); - assertEquals(fieldAccessor.getPropertyValue("key"),"someKey"); - assertEquals(fieldAccessor.getPropertyValue("payload"),"somePayload"); - assertSame(fieldAccessor.getPropertyValue("producerListener"), producerListener); - verifyNoMoreInteractions(producer); - } - - @Test - @SuppressWarnings({"unchecked","rawtypes"}) - public void testProducerListenerNotSet() throws Exception { - KafkaProducerContext producerContext = new KafkaProducerContext(); - Serializer serializer = Mockito.mock(Serializer.class); - ProducerMetadata producerMetadata - = new ProducerMetadata<>("default", Object.class, Object.class, serializer, serializer); - Producer producer = Mockito.mock(Producer.class); - ProducerConfiguration producerConfiguration - = new ProducerConfiguration<>(producerMetadata, producer); - Map> producerConfigurations - = Collections.>singletonMap("default", producerConfiguration); - producerContext.setProducerConfigurations(producerConfigurations); - KafkaProducerMessageHandler handler = new KafkaProducerMessageHandler(producerContext); - handler.handleMessage(MessageBuilder.withPayload("somePayload").build()); - verify(producer).send(any(ProducerRecord.class)); - verifyNoMoreInteractions(producer); - } -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/outbound/ProducerTest.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/outbound/ProducerTest.java deleted file mode 100644 index 9d717c3727..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/outbound/ProducerTest.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.outbound; - -import org.apache.kafka.clients.producer.Producer; -import org.apache.kafka.common.serialization.Serializer; -import org.junit.Test; -import org.mockito.Mockito; - -import org.springframework.core.convert.ConversionService; -import org.springframework.integration.kafka.support.ProducerConfiguration; -import org.springframework.integration.kafka.support.ProducerMetadata; - -/** - * @author Marius Bogoevici - */ -public class ProducerTest { - - @Test - @SuppressWarnings({"unchecked","rawtypes"}) - public void producerContextDataConversionTest() { - Producer producer = (Producer) Mockito.mock(Producer.class); - Serializer serializer = Mockito.mock(Serializer.class); - ProducerMetadata producerMetadata = - new ProducerMetadata<>("topic", FooBase.class, BarBase.class, serializer,serializer); - ProducerConfiguration producerConfiguration = - new ProducerConfiguration<>(producerMetadata,producer); - ConversionService conversionService = Mockito.mock(ConversionService.class); - producerConfiguration.setConversionService(conversionService); - producerConfiguration.convertAndSend("topic",0,new Foo(), new Bar()); - // the key and payload can be cast automatically, no conversion is necessary - Mockito.verifyZeroInteractions(conversionService); - } - - public static class FooBase { - } - - public static class Foo extends FooBase { - } - - public static class BarBase { - } - - public static class Bar extends BarBase { - } -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/performance/OffsetManagerPerformanceTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/performance/OffsetManagerPerformanceTests.java deleted file mode 100644 index fb28c7a6c3..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/performance/OffsetManagerPerformanceTests.java +++ /dev/null @@ -1,154 +0,0 @@ -/* - * Copyright 2016 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.performance; - -import static org.hamcrest.CoreMatchers.is; -import static org.springframework.integration.kafka.util.TopicUtils.ensureTopicCreated; - -import java.util.Properties; - -import org.I0Itec.zkclient.ZkClient; -import org.junit.Assert; -import org.junit.Rule; -import org.junit.Test; - -import org.springframework.beans.factory.DisposableBean; -import org.springframework.beans.factory.InitializingBean; -import org.springframework.integration.kafka.core.DefaultConnectionFactory; -import org.springframework.integration.kafka.core.Partition; -import org.springframework.integration.kafka.core.ZookeeperConfiguration; -import org.springframework.integration.kafka.listener.KafkaNativeOffsetManager; -import org.springframework.integration.kafka.listener.KafkaTopicOffsetManager; -import org.springframework.integration.kafka.listener.OffsetManager; -import org.springframework.integration.kafka.listener.WindowingOffsetManager; -import org.springframework.integration.kafka.rule.KafkaEmbedded; -import org.springframework.integration.kafka.support.ZookeeperConnect; -import org.springframework.util.StopWatch; - -import com.gs.collections.api.RichIterable; -import com.gs.collections.api.block.function.Function2; -import com.gs.collections.api.multimap.Multimap; -import com.gs.collections.api.multimap.MutableMultimap; -import com.gs.collections.api.tuple.Pair; -import com.gs.collections.impl.factory.Multimaps; -import com.gs.collections.impl.tuple.Tuples; -import scala.collection.JavaConversions; -import scala.collection.Map; -import scala.collection.immutable.List$; -import scala.collection.immutable.Map$; -import scala.collection.immutable.Seq; - -/** - * @author Marius Bogoevici - * @author Artem Bilan - */ -public class OffsetManagerPerformanceTests { - - public static final int UPDATE_COUNT = 100000; - - @Rule - public KafkaEmbedded embedded = new KafkaEmbedded(1); - - private final StopWatch stopWatch = new StopWatch("OffsetManagers Performance"); - - @Test - public void testPerformance() throws Exception { - ZookeeperConnect zookeeperConnect = new ZookeeperConnect(embedded.getZookeeperConnectionString()); - - KafkaTopicOffsetManager kafkaTopicOffsetManager = - new KafkaTopicOffsetManager(zookeeperConnect, "zkTopic"); - - KafkaNativeOffsetManager kafkaNativeOffsetManager = new KafkaNativeOffsetManager( - new DefaultConnectionFactory(new ZookeeperConfiguration(zookeeperConnect)), zookeeperConnect); - - KafkaTopicOffsetManager topicOffsetManagerForAssertion = - new KafkaTopicOffsetManager(zookeeperConnect, "zkTopic"); - - KafkaNativeOffsetManager nativeOffsetManagerForAssertion = new KafkaNativeOffsetManager( - new DefaultConnectionFactory(new ZookeeperConfiguration(zookeeperConnect)), zookeeperConnect); - - doTest("Just KafkaTopicOffsetManager", kafkaTopicOffsetManager, topicOffsetManagerForAssertion, false); - doTest("Just KafkaNativeOffsetManager", kafkaNativeOffsetManager, nativeOffsetManagerForAssertion, false); - doTest("Window KafkaTopicOffsetManager", kafkaTopicOffsetManager, topicOffsetManagerForAssertion, true); - doTest("Window KafkaNativeOffsetManager", kafkaNativeOffsetManager, nativeOffsetManagerForAssertion, true); - - System.out.println(this.stopWatch.prettyPrint()); - } - - private void doTest(String description, OffsetManager offsetManager, - OffsetManager offsetManagerForAssertion, boolean window) throws Exception { - createTopic(this.embedded.getZkClient(), "sometopic", 1, 1, 1); - Partition partition = new Partition("sometopic", 0); - - ((InitializingBean) offsetManager).afterPropertiesSet(); - - offsetManager.updateOffset(partition, 0); - - if (window) { - WindowingOffsetManager windowingOffsetManager = new WindowingOffsetManager(offsetManager); - windowingOffsetManager.setCount(100); - windowingOffsetManager.afterPropertiesSet(); - offsetManager = windowingOffsetManager; - } - - this.stopWatch.start(description); - for (long i = 1; i < UPDATE_COUNT; i++) { - offsetManager.updateOffset(partition, i); - } - stopWatch.stop(); - ((DisposableBean) offsetManager).destroy(); - - ((InitializingBean) offsetManagerForAssertion).afterPropertiesSet(); - Assert.assertThat(offsetManagerForAssertion.getOffset(partition), is(99999L)); - } - - @SuppressWarnings("unchecked") - public void createTopic(ZkClient zkClient, String topicName, int partitionCount, int brokers, int replication) { - MutableMultimap partitionDistribution = - createPartitionDistribution(partitionCount, brokers, replication); - ensureTopicCreated(zkClient, topicName, partitionCount, new Properties(), - toKafkaPartitionMap(partitionDistribution)); - } - - - public MutableMultimap createPartitionDistribution(int partitionCount, int brokers, - int replication) { - MutableMultimap partitionDistribution = Multimaps.mutable.list.with(); - for (int i = 0; i < partitionCount; i++) { - for (int j = 0; j < replication; j++) { - partitionDistribution.put(i, (i + j) % brokers); - } - } - return partitionDistribution; - } - - @SuppressWarnings({"rawtypes", "serial", "deprecation", "unchecked"}) - private Map toKafkaPartitionMap(Multimap partitions) { - java.util.Map> m = partitions.toMap() - .collect(new Function2, Pair>>() { - - @Override - public Pair> value(Integer argument1, RichIterable argument2) { - return Tuples.pair((Object) argument1, - List$.MODULE$.fromArray(argument2.toArray(new Object[0])).toSeq()); - } - - }); - return Map$.MODULE$.apply(JavaConversions.asScalaMap(m).toSeq()); - } - -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/rule/KafkaEmbedded.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/rule/KafkaEmbedded.java deleted file mode 100644 index 6fd52c2b08..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/rule/KafkaEmbedded.java +++ /dev/null @@ -1,311 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package org.springframework.integration.kafka.rule; - -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.List; -import java.util.Properties; - -import com.gs.collections.api.block.function.Function; -import com.gs.collections.impl.list.mutable.FastList; -import com.gs.collections.impl.utility.ListIterate; -import kafka.admin.AdminUtils$; -import kafka.api.PartitionMetadata; -import kafka.api.TopicMetadata; -import kafka.cluster.Broker; -import kafka.server.KafkaConfig; -import kafka.server.KafkaServer; -import kafka.server.NotRunning; -import kafka.utils.SystemTime$; -import kafka.utils.TestUtils; -import kafka.utils.TestZKUtils; -import kafka.utils.Utils; -import kafka.utils.ZKStringSerializer$; -import kafka.zk.EmbeddedZookeeper; -import org.I0Itec.zkclient.ZkClient; -import org.I0Itec.zkclient.exception.ZkInterruptedException; -import org.apache.kafka.common.protocol.Errors; -import org.junit.rules.ExternalResource; -import scala.collection.JavaConversions; -import scala.collection.Map; -import scala.collection.Set; - -import org.springframework.integration.kafka.core.BrokerAddress; -import org.springframework.retry.RetryCallback; -import org.springframework.retry.RetryContext; -import org.springframework.retry.backoff.ExponentialBackOffPolicy; -import org.springframework.retry.policy.SimpleRetryPolicy; -import org.springframework.retry.support.RetryTemplate; - -/** - * @author Marius Bogoevici - * @author Artem Bilan - */ -@SuppressWarnings("serial") -public class KafkaEmbedded extends ExternalResource implements KafkaRule { - - public static final long METADATA_PROPAGATION_TIMEOUT = 10000L; - - private int count; - - private boolean controlledShutdown; - - private List kafkaPorts; - - private List kafkaServers; - - private EmbeddedZookeeper zookeeper; - - private ZkClient zookeeperClient; - - @SuppressWarnings("unchecked") - public KafkaEmbedded(int count, boolean controlledShutdown) { - this.count = count; - this.controlledShutdown = controlledShutdown; - this.kafkaPorts = JavaConversions.asJavaList((scala.collection.immutable.List) TestUtils.choosePorts(count)); - } - - public KafkaEmbedded(int count) { - this(count, false); - } - - @Override - protected void before() throws Throwable { - startZookeeper(); - int zkConnectionTimeout = 6000; - int zkSessionTimeout = 6000; - zookeeperClient = new ZkClient(TestZKUtils.zookeeperConnect(), zkSessionTimeout, zkConnectionTimeout, - ZKStringSerializer$.MODULE$); - kafkaServers = new ArrayList(); - for (int i = 0; i < count; i++) { - Properties brokerConfigProperties = TestUtils.createBrokerConfig(i, kafkaPorts.get(i),controlledShutdown); - brokerConfigProperties.setProperty("replica.socket.timeout.ms","1000"); - brokerConfigProperties.setProperty("controller.socket.timeout.ms","1000"); - brokerConfigProperties.setProperty("offsets.topic.replication.factor","1"); - KafkaServer server = TestUtils.createServer(new KafkaConfig(brokerConfigProperties), SystemTime$.MODULE$); - kafkaServers.add(server); - } - } - - @Override - protected void after() { - for (KafkaServer kafkaServer : kafkaServers) { - try { - if (kafkaServer.brokerState().currentState() != (NotRunning.state())) { - kafkaServer.shutdown(); - kafkaServer.awaitShutdown(); - } - } - catch (Exception e) { - // do nothing - } - try { - Utils.rm(kafkaServer.config().logDirs()); - } - catch (Exception e) { - // do nothing - } - } - try { - zookeeperClient.close(); - } - catch (ZkInterruptedException e) { - // do nothing - } - try { - zookeeper.shutdown(); - } - catch (Exception e) { - // do nothing - } - } - - public List getKafkaServers() { - return kafkaServers; - } - - public KafkaServer getKafkaServer(int id) { - return kafkaServers.get(id); - } - - public EmbeddedZookeeper getZookeeper() { - return zookeeper; - } - - @Override - public ZkClient getZkClient() { - return zookeeperClient; - } - - @Override - public String getZookeeperConnectionString() { - return zookeeper.connectString(); - } - - public BrokerAddress getBrokerAddress(int i) { - KafkaServer kafkaServer = this.kafkaServers.get(i); - return new BrokerAddress(kafkaServer.config().hostName(),kafkaServer.config().port()); - } - - @Override - public BrokerAddress[] getBrokerAddresses() { - return ListIterate.collect(this.kafkaServers, - new Function() { - - @Override - public BrokerAddress valueOf(KafkaServer kafkaServer) { - return new BrokerAddress(kafkaServer.config().hostName(), kafkaServer.config().port()); - } - - }) - .toArray(new BrokerAddress[this.kafkaServers.size()]); - } - - public void bounce(BrokerAddress brokerAddress) { - for (KafkaServer kafkaServer : getKafkaServers()) { - if (brokerAddress.equals(new BrokerAddress(kafkaServer.config().hostName(), kafkaServer.config().port()))) { - kafkaServer.shutdown(); - kafkaServer.awaitShutdown(); - } - } - } - - public void startZookeeper() { - zookeeper = new EmbeddedZookeeper(TestZKUtils.zookeeperConnect()); - } - - public void bounce(int index, boolean waitForPropagation) { - kafkaServers.get(index).shutdown(); - if (waitForPropagation) { - long initialTime = System.currentTimeMillis(); - boolean canExit = false; - do { - try { - Thread.sleep(100); - } - catch (InterruptedException e) { - break; - } - canExit = true; - Map topicProperties = AdminUtils$.MODULE$.fetchAllTopicConfigs(getZkClient()); - Set topicMetadatas = - AdminUtils$.MODULE$.fetchTopicMetadataFromZk(topicProperties.keySet(), getZkClient()); - for (TopicMetadata topicMetadata : JavaConversions.asJavaCollection(topicMetadatas)) { - if (Errors.forCode(topicMetadata.errorCode()).exception() == null) { - for (PartitionMetadata partitionMetadata : - JavaConversions.asJavaCollection(topicMetadata.partitionsMetadata())) { - Collection inSyncReplicas = JavaConversions.asJavaCollection(partitionMetadata.isr()); - for (Broker broker : inSyncReplicas) { - if (broker.id() == index) { - canExit = false; - } - } - } - } - } - } - while (!canExit && (System.currentTimeMillis() - initialTime < METADATA_PROPAGATION_TIMEOUT)); - } - - } - - public void bounce(int index) { - bounce(index, true); - } - - public void restart(final int index) throws Exception { - - // retry restarting repeatedly, first attempts may fail - - SimpleRetryPolicy retryPolicy = new SimpleRetryPolicy(10, - Collections.,Boolean>singletonMap(Exception.class, true)); - - ExponentialBackOffPolicy backOffPolicy = new ExponentialBackOffPolicy(); - backOffPolicy.setInitialInterval(100); - backOffPolicy.setMaxInterval(1000); - backOffPolicy.setMultiplier(2); - - RetryTemplate retryTemplate = new RetryTemplate(); - retryTemplate.setRetryPolicy(retryPolicy); - retryTemplate.setBackOffPolicy(backOffPolicy); - - - retryTemplate.execute(new RetryCallback() { - @Override - public Void doWithRetry(RetryContext context) throws Exception { - kafkaServers.get(index).startup(); - return null; - } - }); - } - - public void waitUntilSynced(String topic, int brokerId) { - long initialTime = System.currentTimeMillis(); - boolean canExit = false; - do { - try { - Thread.sleep(100); - } - catch (InterruptedException e) { - break; - } - canExit = true; - TopicMetadata topicMetadata = AdminUtils$.MODULE$.fetchTopicMetadataFromZk(topic, getZkClient()); - if (Errors.forCode(topicMetadata.errorCode()).exception() == null) { - for (PartitionMetadata partitionMetadata : - JavaConversions.asJavaCollection(topicMetadata.partitionsMetadata())) { - Collection isr = JavaConversions.asJavaCollection(partitionMetadata.isr()); - boolean containsIndex = false; - for (Broker broker : isr) { - if (broker.id() == brokerId) { - containsIndex = true; - } - } - if (!containsIndex) { - canExit = false; - } - - } - } - } - while (!canExit && (System.currentTimeMillis() - initialTime < METADATA_PROPAGATION_TIMEOUT)); - } - - @Override - public String getBrokersAsString() { - return FastList.newList(Arrays.asList(getBrokerAddresses())) - .collect(new Function() { - - @Override - public String valueOf(BrokerAddress object) { - return object.getHost() + ":" + object.getPort(); - } - - }) - .makeString(","); - } - - @Override - public boolean isEmbedded() { - return true; - } - -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/rule/KafkaRule.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/rule/KafkaRule.java deleted file mode 100644 index 8b3d5e551f..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/rule/KafkaRule.java +++ /dev/null @@ -1,46 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - - -package org.springframework.integration.kafka.rule; - -import java.util.List; - -import kafka.server.KafkaServer; -import org.I0Itec.zkclient.ZkClient; -import org.junit.rules.TestRule; - -import org.springframework.integration.kafka.core.BrokerAddress; - -/** - * Common functionality for the Kafka JUnit rules - * - * @author Marius Bogoevici - */ -public interface KafkaRule extends TestRule { - - ZkClient getZkClient(); - - String getZookeeperConnectionString(); - - BrokerAddress[] getBrokerAddresses(); - - String getBrokersAsString(); - - boolean isEmbedded(); - - List getKafkaServers(); -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/rule/KafkaRunning.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/rule/KafkaRunning.java deleted file mode 100644 index 8cc053df63..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/rule/KafkaRunning.java +++ /dev/null @@ -1,129 +0,0 @@ -/* - * Copyright 2014 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.kafka.rule; - -import java.util.Arrays; -import java.util.List; - -import com.gs.collections.api.block.function.Function; -import com.gs.collections.impl.block.factory.Functions; -import com.gs.collections.impl.list.mutable.FastList; -import com.gs.collections.impl.utility.ListIterate; - -import kafka.cluster.Broker; -import kafka.server.KafkaServer; -import kafka.utils.ZKStringSerializer$; -import kafka.utils.ZkUtils; - -import org.I0Itec.zkclient.ZkClient; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.junit.Assume; -import org.junit.rules.TestWatcher; -import org.junit.runner.Description; -import org.junit.runners.model.Statement; - -import scala.collection.JavaConversions; -import scala.collection.Seq; - -import org.springframework.integration.kafka.core.BrokerAddress; -import org.springframework.integration.kafka.core.ZookeeperConnectDefaults; - -/** - * * A rule that prevents integration tests from failing if the Kafka server is not running or not - * accessible. If the Kafka server is not running in the background all the tests here will simply be skipped because - * of a violated assumption (showing as successful). - * The rule can be declared as static so that it only has to check once for all tests in the enclosing test case, but - * there isn't a lot of overhead in making it non-static. - * - * @author Dave Syer - * @author Artem Bilan - * @author Gary Russell - * @author Marius Bogoevici - * @since 1.0 - */ -public class KafkaRunning extends TestWatcher implements KafkaRule { - - private static final String ZOOKEEPER_CONNECT_STRING = ZookeeperConnectDefaults.ZK_CONNECT; - - private static final Log logger = LogFactory.getLog(KafkaRunning.class); - - private ZkClient zkClient; - - /** - * @return a new rule that assumes an existing running broker - */ - public static KafkaRunning isRunning() { - return new KafkaRunning(); - } - - public ZkClient getZkClient() { - return zkClient; - } - - @Override - public String getZookeeperConnectionString() { - return ZOOKEEPER_CONNECT_STRING; - } - - @Override - @SuppressWarnings("serial") - public BrokerAddress[] getBrokerAddresses() { - Seq allBrokersInCluster = ZkUtils.getAllBrokersInCluster(zkClient); - return ListIterate.collect(JavaConversions.asJavaList(allBrokersInCluster), - new Function() { - - @Override - public BrokerAddress valueOf(Broker broker) { - return new BrokerAddress(broker.host(), broker.port()); - } - - }) - .toArray(new BrokerAddress[allBrokersInCluster.size()]); - } - - @Override - public String getBrokersAsString() { - return FastList.newList(Arrays.asList(getBrokerAddresses())).collect(Functions.getToString()).makeString(","); - } - - @Override - public boolean isEmbedded() { - return false; - } - - @Override - public List getKafkaServers() { - throw new UnsupportedOperationException("Not supported on the external rule"); - } - - @Override - public Statement apply(Statement base, Description description) { - try { - this.zkClient = new ZkClient(ZOOKEEPER_CONNECT_STRING, 1000, 1000, ZKStringSerializer$.MODULE$); - if (getBrokerAddresses().length == 0) { - throw new IllegalStateException("No running Kafka brokers"); - } - } - catch (Exception e) { - logger.warn("Not executing tests because basic connectivity test failed"); - Assume.assumeNoException(e); - } - - return super.apply(base, description); - } - -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/serializer/AvroReflectDatumBackedKafkaSerializerTest.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/serializer/AvroReflectDatumBackedKafkaSerializerTest.java deleted file mode 100644 index b374d92348..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/serializer/AvroReflectDatumBackedKafkaSerializerTest.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.kafka.serializer; - -import org.junit.Assert; -import org.junit.Test; -import org.springframework.integration.kafka.serializer.avro.AvroReflectDatumBackedKafkaDecoder; -import org.springframework.integration.kafka.serializer.avro.AvroReflectDatumBackedKafkaEncoder; -import org.springframework.integration.kafka.test.utils.TestObject; - -/** - * @author Soby Chacko - * @since 0.5 - */ -public class AvroReflectDatumBackedKafkaSerializerTest { - - @Test - public void testDecodePlainSchema() { - final AvroReflectDatumBackedKafkaEncoder avroBackedKafkaEncoder = new AvroReflectDatumBackedKafkaEncoder(TestObject.class); - - final TestObject testObject = new TestObject(); - testObject.setTestData1("\"Test Data1\""); - testObject.setTestData2(1); - - final byte[] data = avroBackedKafkaEncoder.toBytes(testObject); - - final AvroReflectDatumBackedKafkaDecoder avroReflectDatumBackedKafkaDecoder = new AvroReflectDatumBackedKafkaDecoder(TestObject.class); - final TestObject decodedFbu = avroReflectDatumBackedKafkaDecoder.fromBytes(data); - - Assert.assertEquals(testObject.getTestData1(), decodedFbu.getTestData1()); - Assert.assertEquals(testObject.getTestData2(), decodedFbu.getTestData2()); - } - - @Test - public void anotherTest() { - final AvroReflectDatumBackedKafkaEncoder avroBackedKafkaEncoder = new AvroReflectDatumBackedKafkaEncoder(java.lang.String.class); - final String testString = "Testing Avro"; - final byte[] data = avroBackedKafkaEncoder.toBytes(testString); - - final AvroReflectDatumBackedKafkaDecoder avroReflectDatumBackedKafkaDecoder = new AvroReflectDatumBackedKafkaDecoder(java.lang.String.class); - final String decodedS = avroReflectDatumBackedKafkaDecoder.fromBytes(data); - - Assert.assertEquals(testString, decodedS); - } -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/serializer/AvroSpecificDatumBackedKafkaSerializerTest.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/serializer/AvroSpecificDatumBackedKafkaSerializerTest.java deleted file mode 100644 index 3850a5ab47..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/serializer/AvroSpecificDatumBackedKafkaSerializerTest.java +++ /dev/null @@ -1,29 +0,0 @@ -package org.springframework.integration.kafka.serializer; - -import org.junit.Assert; -import org.junit.Test; -import org.springframework.integration.kafka.serializer.avro.AvroSpecificDatumBackedKafkaDecoder; -import org.springframework.integration.kafka.serializer.avro.AvroSpecificDatumBackedKafkaEncoder; -import org.springframework.integration.kafka.test.utils.User; - -/** - * @author Soby Chacko - * @since 0.5 - */ -public class AvroSpecificDatumBackedKafkaSerializerTest { - - @Test - public void testEncodeDecodeFromSpecificDatumSchema() { - final AvroSpecificDatumBackedKafkaEncoder avroBackedKafkaEncoder = new AvroSpecificDatumBackedKafkaEncoder(User.class); - - final User user = new User("First", "Last"); - - final byte[] data = avroBackedKafkaEncoder.toBytes(user); - - final AvroSpecificDatumBackedKafkaDecoder avroSpecificDatumBackedKafkaDecoder = new AvroSpecificDatumBackedKafkaDecoder(User.class); - final User decodedUser = avroSpecificDatumBackedKafkaDecoder.fromBytes(data); - - Assert.assertEquals(user.getFirstName(), decodedUser.getFirstName().toString()); - Assert.assertEquals(user.getLastName(), decodedUser.getLastName().toString()); - } -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/support/ConsumerConfigurationTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/support/ConsumerConfigurationTests.java deleted file mode 100644 index f7fdd6ea9b..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/support/ConsumerConfigurationTests.java +++ /dev/null @@ -1,538 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.kafka.support; - -import static org.junit.Assert.assertNull; -import static org.mockito.Mockito.atLeast; -import static org.mockito.Mockito.atMost; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import java.util.ArrayList; -import java.util.Collection; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.junit.Assert; -import org.junit.Test; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; - -import kafka.consumer.ConsumerIterator; -import kafka.consumer.KafkaStream; -import kafka.javaapi.consumer.ConsumerConnector; -import kafka.message.MessageAndMetadata; -import kafka.serializer.Decoder; - -/** - * @author Soby Chacko - * @author Rajasekar Elango - * @since 0.5 - */ -@Deprecated -@SuppressWarnings("deprecation") -public class ConsumerConfigurationTests { - - @Test - @SuppressWarnings("unchecked") - public void testReceiveMessageForSingleTopicFromSingleStream() { - final ConsumerMetadata consumerMetadata = mock(ConsumerMetadata.class); - final ConsumerConnectionProvider consumerConnectionProvider = - mock(ConsumerConnectionProvider.class); - final MessageLeftOverTracker messageLeftOverTracker = mock(MessageLeftOverTracker.class); - final ConsumerConnector consumerConnector = mock(ConsumerConnector.class); - - Map topicStreamMap = new HashMap(); - topicStreamMap.put("topic1", 1); - when(consumerMetadata.getTopicStreamMap()).thenReturn(topicStreamMap); - - when(consumerConnectionProvider.getConsumerConnector()).thenReturn(consumerConnector); - - final ConsumerConfiguration consumerConfiguration = new ConsumerConfiguration(consumerMetadata, - consumerConnectionProvider, messageLeftOverTracker); - consumerConfiguration.setMaxMessages(1); - - final KafkaStream stream = mock(KafkaStream.class); - final List> streams = new ArrayList>(); - streams.add(stream); - final Map>> messageStreams = new HashMap>>(); - messageStreams.put("topic", streams); - - when(consumerConfiguration.createMessageStreamsForTopic()).thenReturn(messageStreams); - final ConsumerIterator iterator = mock(ConsumerIterator.class); - when(stream.iterator()).thenReturn(iterator); - final MessageAndMetadata messageAndMetadata = mock(MessageAndMetadata.class); - when(iterator.next()).thenReturn(messageAndMetadata); - when(messageAndMetadata.message()).thenReturn((V) "got message"); - when(messageAndMetadata.topic()).thenReturn("topic"); - when(messageAndMetadata.partition()).thenReturn(1); - - final Map>> messages = consumerConfiguration.receive(); - Assert.assertEquals(1, messages.size()); - Assert.assertEquals(1, messages.get("topic").size()); - Assert.assertEquals("got message", messages.get("topic").get(1).get(0)); - - verify(stream, times(1)).iterator(); - verify(iterator, times(1)).next(); - verify(messageAndMetadata, times(1)).message(); - verify(messageAndMetadata, times(1)).topic(); - } - - @Test - @SuppressWarnings("unchecked") - public void testReceiveMessageForSingleTopicFromMultipleStreams() { - final ConsumerMetadata consumerMetadata = mock(ConsumerMetadata.class); - final ConsumerConnectionProvider consumerConnectionProvider = - mock(ConsumerConnectionProvider.class); - final MessageLeftOverTracker messageLeftOverTracker = mock(MessageLeftOverTracker.class); - - Map topicStreamMap = new HashMap(); - topicStreamMap.put("topic1", 1); - when(consumerMetadata.getTopicStreamMap()).thenReturn(topicStreamMap); - - final ConsumerConnector consumerConnector = mock(ConsumerConnector.class); - - when(consumerConnectionProvider.getConsumerConnector()).thenReturn(consumerConnector); - - final ConsumerConfiguration consumerConfiguration = new ConsumerConfiguration(consumerMetadata, - consumerConnectionProvider, messageLeftOverTracker); - consumerConfiguration.setMaxMessages(3); - - final KafkaStream stream1 = mock(KafkaStream.class); - final KafkaStream stream2 = mock(KafkaStream.class); - final KafkaStream stream3 = mock(KafkaStream.class); - final List> streams = new ArrayList>(); - streams.add(stream1); - streams.add(stream2); - streams.add(stream3); - final Map>> messageStreams = new HashMap>>(); - messageStreams.put("topic", streams); - - when(consumerConfiguration.createMessageStreamsForTopic()).thenReturn(messageStreams); - final ConsumerIterator iterator1 = mock(ConsumerIterator.class); - final ConsumerIterator iterator2 = mock(ConsumerIterator.class); - final ConsumerIterator iterator3 = mock(ConsumerIterator.class); - - when(stream1.iterator()).thenReturn(iterator1); - when(stream2.iterator()).thenReturn(iterator2); - when(stream3.iterator()).thenReturn(iterator3); - final MessageAndMetadata messageAndMetadata1 = mock(MessageAndMetadata.class); - final MessageAndMetadata messageAndMetadata2 = mock(MessageAndMetadata.class); - final MessageAndMetadata messageAndMetadata3 = mock(MessageAndMetadata.class); - - when(iterator1.next()).thenReturn(messageAndMetadata1); - when(iterator2.next()).thenReturn(messageAndMetadata2); - when(iterator3.next()).thenReturn(messageAndMetadata3); - - when(messageAndMetadata1.message()).thenReturn((V)"got message"); - when(messageAndMetadata1.topic()).thenReturn("topic"); - when(messageAndMetadata1.partition()).thenReturn(1); - - when(messageAndMetadata2.message()).thenReturn((V)"got message"); - when(messageAndMetadata2.topic()).thenReturn("topic"); - when(messageAndMetadata2.partition()).thenReturn(2); - - when(messageAndMetadata3.message()).thenReturn((V)"got message"); - when(messageAndMetadata3.topic()).thenReturn("topic"); - when(messageAndMetadata3.partition()).thenReturn(3); - - final Map>> messages = consumerConfiguration.receive(); - Assert.assertEquals(messages.size(), 1); - int sum = 0; - - final Map> values = messages.get("topic"); - - for (final List l : values.values()) { - sum += l.size(); - } - - Assert.assertEquals(3, sum); - } - - @Test - @SuppressWarnings("unchecked") - public void testReceiveMessageForMultipleTopicsFromMultipleStreams() { - final ConsumerMetadata consumerMetadata = mock(ConsumerMetadata.class); - final ConsumerConnectionProvider consumerConnectionProvider = - mock(ConsumerConnectionProvider.class); - final MessageLeftOverTracker messageLeftOverTracker = mock(MessageLeftOverTracker.class); - - Map topicStreamMap = new HashMap(); - topicStreamMap.put("topic1", 1); - when(consumerMetadata.getTopicStreamMap()).thenReturn(topicStreamMap); - - - final ConsumerConnector consumerConnector = mock(ConsumerConnector.class); - - when(consumerConnectionProvider.getConsumerConnector()).thenReturn(consumerConnector); - - final ConsumerConfiguration consumerConfiguration = new ConsumerConfiguration(consumerMetadata, - consumerConnectionProvider, messageLeftOverTracker); - consumerConfiguration.setMaxMessages(9); - - final KafkaStream stream1 = mock(KafkaStream.class); - final KafkaStream stream2 = mock(KafkaStream.class); - final KafkaStream stream3 = mock(KafkaStream.class); - final List> streams = new ArrayList>(); - streams.add(stream1); - streams.add(stream2); - streams.add(stream3); - final Map>> messageStreams = new HashMap>>(); - messageStreams.put("topic1", streams); - messageStreams.put("topic2", streams); - messageStreams.put("topic3", streams); - - when(consumerConfiguration.createMessageStreamsForTopic()).thenReturn(messageStreams); - final ConsumerIterator iterator1 = mock(ConsumerIterator.class); - final ConsumerIterator iterator2 = mock(ConsumerIterator.class); - final ConsumerIterator iterator3 = mock(ConsumerIterator.class); - - when(stream1.iterator()).thenReturn(iterator1); - when(stream2.iterator()).thenReturn(iterator2); - when(stream3.iterator()).thenReturn(iterator3); - final MessageAndMetadata messageAndMetadata1 = mock(MessageAndMetadata.class); - final MessageAndMetadata messageAndMetadata2 = mock(MessageAndMetadata.class); - final MessageAndMetadata messageAndMetadata3 = mock(MessageAndMetadata.class); - - when(iterator1.next()).thenReturn(messageAndMetadata1); - when(iterator2.next()).thenReturn(messageAndMetadata2); - when(iterator3.next()).thenReturn(messageAndMetadata3); - - when(messageAndMetadata1.message()).thenReturn((V)"got message1"); - when(messageAndMetadata1.topic()).thenReturn("topic1"); - when(messageAndMetadata1.partition()).thenAnswer(getAnswer()); - - when(messageAndMetadata2.message()).thenReturn((V)"got message2"); - when(messageAndMetadata2.topic()).thenReturn("topic2"); - when(messageAndMetadata1.partition()).thenAnswer(getAnswer()); - - when(messageAndMetadata3.message()).thenReturn((V)"got message3"); - when(messageAndMetadata3.topic()).thenReturn("topic3"); - when(messageAndMetadata1.partition()).thenAnswer(getAnswer()); - - final Map>> messages = consumerConfiguration.receive(); - - int sum = 0; - - final Collection>> values = messages.values(); - - for (final Map> m : values) { - for (final List l : m.values()) { - sum += l.size(); - } - } - - Assert.assertEquals(9, sum); - } - - - - private Answer getAnswer() { - return new Answer() { - private int count = 0; - - @Override - public Object answer(final InvocationOnMock invocation) throws Throwable { - if (count++ == 1) { - return 1; - } else if (count++ == 2) { - return 2; - } - - return 3; - } - }; - } - - @Test - @SuppressWarnings("unchecked") - public void testReceiveMessageAndVerifyMessageLeftoverFromPreviousPollAreTakenFirst() { - final ConsumerMetadata consumerMetadata = mock(ConsumerMetadata.class); - final ConsumerConnectionProvider consumerConnectionProvider = - mock(ConsumerConnectionProvider.class); - final MessageLeftOverTracker messageLeftOverTracker = mock(MessageLeftOverTracker.class); - final ConsumerConnector consumerConnector = mock(ConsumerConnector.class); - - Map topicStreamMap = new HashMap(); - topicStreamMap.put("topic1", 1); - when(consumerMetadata.getTopicStreamMap()).thenReturn(topicStreamMap); - when(messageLeftOverTracker.getCurrentCount()).thenReturn(3); - - final MessageAndMetadata m1 = mock(MessageAndMetadata.class); - final MessageAndMetadata m2 = mock(MessageAndMetadata.class); - final MessageAndMetadata m3 = mock(MessageAndMetadata.class); - - when(m1.key()).thenReturn("key1"); - when(m1.message()).thenReturn("value1"); - when(m1.topic()).thenReturn("topic1"); - when(m1.partition()).thenReturn(1); - - when(m2.key()).thenReturn("key2"); - when(m2.message()).thenReturn("value2"); - when(m2.topic()).thenReturn("topic2"); - when(m2.partition()).thenReturn(1); - - when(m3.key()).thenReturn("key1"); - when(m3.message()).thenReturn("value3"); - when(m3.topic()).thenReturn("topic3"); - when(m3.partition()).thenReturn(1); - - final List> mList = new ArrayList>(); - mList.add(m1); - mList.add(m2); - mList.add(m3); - - when((List>) (Object) messageLeftOverTracker.getMessageLeftOverFromPreviousPoll()).thenReturn(mList); - - when(consumerConnectionProvider.getConsumerConnector()).thenReturn(consumerConnector); - - final ConsumerConfiguration consumerConfiguration = new ConsumerConfiguration(consumerMetadata, - consumerConnectionProvider, messageLeftOverTracker); - consumerConfiguration.setMaxMessages(5); - - final KafkaStream stream = mock(KafkaStream.class); - final List> streams = new ArrayList>(); - streams.add(stream); - final Map>> messageStreams = new HashMap>>(); - messageStreams.put("topic1", streams); - when(consumerConfiguration.createMessageStreamsForTopic()).thenReturn(messageStreams); - final ConsumerIterator iterator = mock(ConsumerIterator.class); - when(stream.iterator()).thenReturn(iterator); - final MessageAndMetadata messageAndMetadata = mock(MessageAndMetadata.class); - when(iterator.next()).thenReturn(messageAndMetadata); - when(messageAndMetadata.message()).thenReturn((V) "got message"); - when(messageAndMetadata.topic()).thenReturn("topic1"); - when(messageAndMetadata.partition()).thenReturn(1); - - final Map>> messages = consumerConfiguration.receive(); - int sum = 0; - - final Collection>> values = messages.values(); - - for (final Map> m : values) { - for (final List l : m.values()) { - sum += l.size(); - } - - } - Assert.assertEquals(5, sum); - - Assert.assertTrue(messages.containsKey("topic1")); - Assert.assertTrue(messages.containsKey("topic2")); - Assert.assertTrue(messages.containsKey("topic3")); - - Assert.assertTrue(valueFound(messages.get("topic1").get(1), "value1")); - Assert.assertTrue(valueFound(messages.get("topic2").get(1), "value2")); - Assert.assertTrue(valueFound(messages.get("topic3").get(1), "value3")); - } - - @Test - @SuppressWarnings("unchecked") - public void testGetConsumerMapWithMessageStreamsWithNullDecoders() { - - final ConsumerMetadata mockedConsumerMetadata = mock(ConsumerMetadata.class); - - assertNull(mockedConsumerMetadata.getKeyDecoder()); - assertNull(mockedConsumerMetadata.getValueDecoder()); - - final Map topicsStreamMap = new HashMap(); - when(mockedConsumerMetadata.getTopicStreamMap()).thenReturn(topicsStreamMap); - - final ConsumerConnectionProvider mockedConsumerConnectionProvider = mock(ConsumerConnectionProvider.class); - final MessageLeftOverTracker mockedMessageLeftOverTracker = mock(MessageLeftOverTracker.class); - final ConsumerConnector mockedConsumerConnector = mock(ConsumerConnector.class); - - when(mockedConsumerConnectionProvider.getConsumerConnector()).thenReturn(mockedConsumerConnector); - - final Map>> messageStreams = new HashMap>>(); - when((Map>>) - (Object) mockedConsumerConnector.createMessageStreams(topicsStreamMap)).thenReturn(messageStreams); - - final ConsumerConfiguration consumerConfiguration = new ConsumerConfiguration(mockedConsumerMetadata, - mockedConsumerConnectionProvider, mockedMessageLeftOverTracker); - - consumerConfiguration.createMessageStreamsForTopic(); - - verify(mockedConsumerMetadata, atLeast(1)).getTopicStreamMap(); - verify(mockedConsumerConnector, atLeast(1)).createMessageStreams(topicsStreamMap, null, null); - //verify(mockedConsumerConnector, atMost(0)).createMessageStreams(topicsStreamMap, null, null); - } - - @Test - @SuppressWarnings("unchecked") - public void testGetConsumerMapWithMessageStreamsWithDecoders() { - - @SuppressWarnings("unchecked") - final ConsumerMetadata mockedConsumerMetadata = mock(ConsumerMetadata.class); - - final Map topicsStreamMap = new HashMap(); - when(mockedConsumerMetadata.getTopicStreamMap()).thenReturn(topicsStreamMap); - - @SuppressWarnings("unchecked") - final Decoder mockedKeyDecoder = mock(Decoder.class); - - @SuppressWarnings("unchecked") - final Decoder mockedValueDecoder = mock(Decoder.class); - - when(mockedConsumerMetadata.getKeyDecoder()).thenReturn(mockedKeyDecoder); - when(mockedConsumerMetadata.getValueDecoder()).thenReturn(mockedValueDecoder); - - final ConsumerConnectionProvider mockedConsumerConnectionProvider = - mock(ConsumerConnectionProvider.class); - final MessageLeftOverTracker mockedMessageLeftOverTracker = mock(MessageLeftOverTracker.class); - final ConsumerConnector mockedConsumerConnector = mock(ConsumerConnector.class); - - when(mockedConsumerConnectionProvider.getConsumerConnector()).thenReturn(mockedConsumerConnector); - - final Map>> messageStreams = new HashMap>>(); - when(mockedConsumerConnector.createMessageStreams(topicsStreamMap)).thenReturn(messageStreams); - - final ConsumerConfiguration consumerConfiguration = - new ConsumerConfiguration(mockedConsumerMetadata, mockedConsumerConnectionProvider, - mockedMessageLeftOverTracker); - - consumerConfiguration.createMessageStreamsForTopic(); - - verify(mockedConsumerMetadata, atLeast(1)).getTopicStreamMap(); - verify(mockedConsumerConnector, atMost(0)).createMessageStreams(topicsStreamMap); - verify(mockedConsumerConnector, atLeast(1)) - .createMessageStreams(topicsStreamMap, mockedKeyDecoder, mockedValueDecoder); - } - - - @Test - @SuppressWarnings("unchecked") - public void testReceiveMessageForTopicFilterFromSingleStream() { - final ConsumerMetadata consumerMetadata = mock(ConsumerMetadata.class); - final ConsumerConnectionProvider consumerConnectionProvider = - mock(ConsumerConnectionProvider.class); - final MessageLeftOverTracker messageLeftOverTracker = mock(MessageLeftOverTracker.class); - final ConsumerConnector consumerConnector = mock(ConsumerConnector.class); - - when(consumerMetadata.getTopicFilterConfiguration()).thenReturn(new TopicFilterConfiguration(".*", 1, false)); - - when(consumerConnectionProvider.getConsumerConnector()).thenReturn(consumerConnector); - - final ConsumerConfiguration consumerConfiguration = - new ConsumerConfiguration(consumerMetadata, consumerConnectionProvider, - messageLeftOverTracker); - consumerConfiguration.setMaxMessages(1); - - final KafkaStream stream = mock(KafkaStream.class); - final List> streams = new ArrayList>(); - streams.add(stream); - - when(consumerConfiguration.createMessageStreamsForTopicFilter()).thenReturn(streams); - final ConsumerIterator iterator = mock(ConsumerIterator.class); - when(stream.iterator()).thenReturn(iterator); - final MessageAndMetadata messageAndMetadata = mock(MessageAndMetadata.class); - when(iterator.next()).thenReturn(messageAndMetadata); - when(messageAndMetadata.message()).thenReturn("got message"); - when(messageAndMetadata.topic()).thenReturn("topic"); - when(messageAndMetadata.partition()).thenReturn(1); - - final Map>> messages = consumerConfiguration.receive(); - Assert.assertEquals(1, messages.size()); - Assert.assertEquals(1, messages.get("topic").size()); - Assert.assertEquals("got message", messages.get("topic").get(1).get(0)); - - verify(stream, times(1)).iterator(); - verify(iterator, times(1)).next(); - verify(messageAndMetadata, times(1)).message(); - verify(messageAndMetadata, times(1)).topic(); - } - - @Test - @SuppressWarnings("unchecked") - public void testReceiveMessageForTopicFilterFromMultipleStreams() { - final ConsumerMetadata consumerMetadata = mock(ConsumerMetadata.class); - final ConsumerConnectionProvider consumerConnectionProvider = - mock(ConsumerConnectionProvider.class); - final MessageLeftOverTracker messageLeftOverTracker = mock(MessageLeftOverTracker.class); - - when(consumerMetadata.getTopicFilterConfiguration()).thenReturn(new TopicFilterConfiguration(".*", 1, false)); - - final ConsumerConnector consumerConnector = mock(ConsumerConnector.class); - - when(consumerConnectionProvider.getConsumerConnector()).thenReturn(consumerConnector); - - final ConsumerConfiguration consumerConfiguration = - new ConsumerConfiguration(consumerMetadata, consumerConnectionProvider, - messageLeftOverTracker); - consumerConfiguration.setMaxMessages(3); - - final KafkaStream stream1 = mock(KafkaStream.class); - final KafkaStream stream2 = mock(KafkaStream.class); - final KafkaStream stream3 = mock(KafkaStream.class); - final List> streams = new ArrayList>(); - streams.add(stream1); - streams.add(stream2); - streams.add(stream3); - - when(consumerConfiguration.createMessageStreamsForTopicFilter()).thenReturn(streams); - final ConsumerIterator iterator1 = mock(ConsumerIterator.class); - final ConsumerIterator iterator2 = mock(ConsumerIterator.class); - final ConsumerIterator iterator3 = mock(ConsumerIterator.class); - - when(stream1.iterator()).thenReturn(iterator1); - when(stream2.iterator()).thenReturn(iterator2); - when(stream3.iterator()).thenReturn(iterator3); - final MessageAndMetadata messageAndMetadata1 = mock(MessageAndMetadata.class); - final MessageAndMetadata messageAndMetadata2 = mock(MessageAndMetadata.class); - final MessageAndMetadata messageAndMetadata3 = mock(MessageAndMetadata.class); - - when(iterator1.next()).thenReturn(messageAndMetadata1); - when(iterator2.next()).thenReturn(messageAndMetadata2); - when(iterator3.next()).thenReturn(messageAndMetadata3); - - when(messageAndMetadata1.message()).thenReturn("got message".getBytes()); - when(messageAndMetadata1.topic()).thenReturn("topic"); - when(messageAndMetadata1.partition()).thenReturn(1); - - when(messageAndMetadata2.message()).thenReturn("got message".getBytes()); - when(messageAndMetadata2.topic()).thenReturn("topic"); - when(messageAndMetadata2.partition()).thenReturn(2); - - when(messageAndMetadata3.message()).thenReturn("got message".getBytes()); - when(messageAndMetadata3.topic()).thenReturn("topic"); - when(messageAndMetadata3.partition()).thenReturn(3); - - final Map>> messages = consumerConfiguration.receive(); - Assert.assertEquals(1, messages.size()); - int sum = 0; - - final Map> values = messages.get("topic"); - - for (final List l : values.values()) { - sum += l.size(); - } - - Assert.assertEquals(3, sum); - } - - private boolean valueFound(final List l, final String value){ - for (final Object o : l){ - if (value.equals(o)){ - return true; - } - } - - return false; - } -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/support/KafkaConsumerContextIntegrationTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/support/KafkaConsumerContextIntegrationTests.java deleted file mode 100644 index ee358928b0..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/support/KafkaConsumerContextIntegrationTests.java +++ /dev/null @@ -1,142 +0,0 @@ -/* - * Copyright 2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.support; - -import static org.hamcrest.Matchers.greaterThanOrEqualTo; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertThat; -import static org.mockito.Mockito.mock; - -import java.util.Collections; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Properties; -import java.util.UUID; - -import org.apache.kafka.common.serialization.StringSerializer; -import org.junit.After; -import org.junit.Rule; -import org.junit.Test; - -import org.springframework.integration.kafka.outbound.KafkaProducerMessageHandler; -import org.springframework.integration.kafka.rule.KafkaEmbedded; -import org.springframework.integration.kafka.rule.KafkaRule; -import org.springframework.integration.kafka.serializer.common.StringDecoder; -import org.springframework.integration.kafka.util.TopicUtils; -import org.springframework.messaging.Message; -import org.springframework.messaging.support.MessageBuilder; - -import kafka.admin.AdminUtils; -import kafka.consumer.ConsumerConfig; - -/** - * @author Gary Russell - * @since 1.2.2 - */ -@Deprecated -@SuppressWarnings("deprecation") -public class KafkaConsumerContextIntegrationTests { - - private static final String TOPIC = "springIntegrationTestInbound"; - - @Rule - public KafkaRule kafkaRule = new KafkaEmbedded(1); - - @After - public void tearDown() { - AdminUtils.deleteTopic(kafkaRule.getZkClient(), TOPIC); - } - - @Test - public void test() throws Exception { - TopicUtils.ensureTopicCreated(kafkaRule.getZookeeperConnectionString(), TOPIC, 1, 1); - - final String suffix = UUID.randomUUID().toString(); - - KafkaProducerContext producerContext = createProducerContext(); - KafkaProducerMessageHandler handler = - new KafkaProducerMessageHandler(producerContext); - - Message msg = MessageBuilder.withPayload("foo" + suffix) - .setHeader(KafkaHeaders.MESSAGE_KEY, "3") - .setHeader(KafkaHeaders.TOPIC, TOPIC) - .build(); - handler.handleMessage(msg); - - KafkaConsumerContext kafkaConsumerContext = createConsumerContext(); - - Message received = kafkaConsumerContext.receive(); - assertNotNull(received); - Map payload = (Map) received.getPayload(); - assertNotNull(payload); - Map topicMap = (Map) payload.get(TOPIC); - assertNotNull(topicMap); - List list = (List) topicMap.get(0); - assertNotNull(list); - assertThat(list.size(), greaterThanOrEqualTo(1)); - assertEquals("foo" + suffix, list.get(0)); - } - - @SuppressWarnings({"rawtypes", "unchecked"}) - private KafkaConsumerContext createConsumerContext() { - KafkaConsumerContext kafkaConsumerContext = new KafkaConsumerContext(); - Map map = new HashMap<>(); - - ConsumerMetadata consumerMetadata = new ConsumerMetadata(); - consumerMetadata.setGroupId("foo"); - consumerMetadata.setValueDecoder(new StringDecoder()); - consumerMetadata.setKeyDecoder(new StringDecoder()); - - Map topicStreamMap = new HashMap<>(); - topicStreamMap.put(TOPIC, 1); - consumerMetadata.setTopicStreamMap(topicStreamMap); - - Properties properties = new Properties(); - properties.put("zookeeper.connect", kafkaRule.getZookeeperConnectionString()); - properties.put("group.id", "foo"); - ConsumerConfig consumerConfig = new ConsumerConfig(properties); - ConsumerConnectionProvider consumerConnectionProvider = new ConsumerConnectionProvider(consumerConfig); - - MessageLeftOverTracker messageLeftOverTracker = mock(MessageLeftOverTracker.class); - ConsumerConfiguration consumerConfiguration = new ConsumerConfiguration(consumerMetadata, - consumerConnectionProvider, messageLeftOverTracker); - map.put("config", consumerConfiguration); - - kafkaConsumerContext.setConsumerConfigurations(map); - return kafkaConsumerContext; - } - - private KafkaProducerContext createProducerContext() throws Exception { - KafkaProducerContext kafkaProducerContext = new KafkaProducerContext(); - ProducerMetadata producerMetadata = new ProducerMetadata<>(TOPIC, String.class, String.class, - new StringSerializer(), new StringSerializer()); - - Properties props = new Properties(); - props.put("linger.ms", "1000"); - ProducerFactoryBean producer = - new ProducerFactoryBean<>(producerMetadata, kafkaRule.getBrokersAsString(), props); - ProducerConfiguration config = - new ProducerConfiguration<>(producerMetadata, producer.getObject()); - Map> producerConfigurationMap = - Collections.>singletonMap(TOPIC, config); - kafkaProducerContext.setProducerConfigurations(producerConfigurationMap); - return kafkaProducerContext; - } - -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/support/KafkaConsumerContextTest.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/support/KafkaConsumerContextTest.java deleted file mode 100644 index c483a1c207..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/support/KafkaConsumerContextTest.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright 2002-2014 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.support; - -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.junit.Assert; -import org.junit.Test; -import org.mockito.Mockito; - -import org.springframework.beans.factory.ListableBeanFactory; -import org.springframework.messaging.Message; - -/** - * @author Soby Chacko - * @since 0.5 - */ -@Deprecated -@SuppressWarnings("deprecation") -public class KafkaConsumerContextTest { - - @Test - @SuppressWarnings("unchecked") - public void testMergeResultsFromMultipleConsumerConfiguration() { - final KafkaConsumerContext kafkaConsumerContext = new KafkaConsumerContext(); - final ListableBeanFactory beanFactory = Mockito.mock(ListableBeanFactory.class); - final ConsumerConfiguration consumerConfiguration1 = Mockito.mock(ConsumerConfiguration.class); - final ConsumerConfiguration consumerConfiguration2 = Mockito.mock(ConsumerConfiguration.class); - - final Map> map = new HashMap>(); - map.put("config1", consumerConfiguration1); - map.put("config2", consumerConfiguration2); - - kafkaConsumerContext.setConsumerConfigurations(map); - - final Map>> result1 = new HashMap>>(); - final List l1 = new ArrayList(); - l1.add("got message1 - l1"); - l1.add("got message2 - l1"); - final Map> innerMap1 = new HashMap>(); - innerMap1.put(1, l1); - result1.put("topic1", innerMap1); - - final Map>> result2 = new HashMap>>(); - final List l2 = new ArrayList(); - l2.add("got message1 - l2"); - l2.add("got message2 - l2"); - l2.add("got message3 - l2"); - - final Map> innerMap2 = new HashMap>(); - innerMap2.put(1, l2); - result1.put("topic2", innerMap2); - - Mockito.when(consumerConfiguration1.receive()).thenReturn(result1); - Mockito.when(consumerConfiguration2.receive()).thenReturn(result2); - - final Message>>> messages = kafkaConsumerContext.receive(); - Assert.assertEquals(messages.getPayload().size(), 2); - Assert.assertEquals(messages.getPayload().get("topic1").size(), 1); - Assert.assertEquals(messages.getPayload().get("topic1").get(1).get(0), "got message1 - l1"); - Assert.assertEquals(messages.getPayload().get("topic1").get(1).get(1), "got message2 - l1"); - - Assert.assertEquals(messages.getPayload().get("topic2").size(), 1); - Assert.assertEquals(messages.getPayload().get("topic2").get(1).get(0), "got message1 - l2"); - Assert.assertEquals(messages.getPayload().get("topic2").get(1).get(1), "got message2 - l2"); - Assert.assertEquals(messages.getPayload().get("topic2").get(1).get(2), "got message3 - l2"); - } - -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/support/KafkaProducerContextTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/support/KafkaProducerContextTests.java deleted file mode 100644 index cf0a80bb78..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/support/KafkaProducerContextTests.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.kafka.support; - -import java.util.HashMap; -import java.util.Map; - -import org.apache.kafka.clients.producer.Producer; -import org.junit.Assert; - -import org.junit.Test; -import org.mockito.Mockito; - -/** - * @author Rajasekar Elango - * @since 0.5 - */ -public class KafkaProducerContextTests { - - @SuppressWarnings({ "rawtypes", "unchecked" }) - @Test - public void testTopicRegexForProducerConfiguration(){ - - final KafkaProducerContext kafkaProducerContext = new KafkaProducerContext(); - - final ProducerMetadata producerMetadata = Mockito.mock(ProducerMetadata.class); - - String testRegex = "test.*"; - - Mockito.when(producerMetadata.getTopic()).thenReturn(testRegex); - final Producer producer = Mockito.mock(Producer.class); - - final ProducerConfiguration producerConfiguration = new ProducerConfiguration(producerMetadata, producer); - - final Map> topicConfigurations = new HashMap>(); - topicConfigurations.put(testRegex, producerConfiguration); - kafkaProducerContext.setProducerConfigurations(topicConfigurations); - - Assert.assertNotNull(kafkaProducerContext.getTopicConfiguration("test1")); - Assert.assertNotNull(kafkaProducerContext.getTopicConfiguration("test2")); - Assert.assertNotNull(kafkaProducerContext.getTopicConfiguration("testabc")); - Assert.assertNull(kafkaProducerContext.getTopicConfiguration("dontmatch_testRegex")); - - } - -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/support/ProducerConfigurationTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/support/ProducerConfigurationTests.java deleted file mode 100644 index 3381221ac8..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/support/ProducerConfigurationTests.java +++ /dev/null @@ -1,345 +0,0 @@ -/* - * Copyright 2013-2014 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.kafka.support; - -import java.io.ByteArrayInputStream; -import java.io.ObjectInputStream; -import java.util.ArrayList; - -import kafka.producer.Partitioner; -import kafka.serializer.DefaultEncoder; - -import org.apache.kafka.clients.producer.Producer; -import org.apache.kafka.clients.producer.ProducerRecord; -import org.apache.kafka.common.PartitionInfo; -import org.apache.kafka.common.serialization.ByteArraySerializer; -import org.apache.kafka.common.serialization.StringSerializer; -import org.junit.Assert; -import org.junit.Test; -import org.mockito.ArgumentCaptor; -import org.mockito.Mockito; -import org.springframework.core.convert.ConversionFailedException; -import org.springframework.integration.kafka.serializer.avro.AvroReflectDatumBackedKafkaEncoder; -import org.springframework.integration.kafka.test.utils.NonSerializableTestKey; -import org.springframework.integration.kafka.test.utils.NonSerializableTestPayload; -import org.springframework.integration.kafka.test.utils.TestKey; -import org.springframework.integration.kafka.test.utils.TestPayload; -import org.springframework.integration.kafka.util.EncoderAdaptingSerializer; -import org.springframework.messaging.Message; -import org.springframework.messaging.support.GenericMessage; - -/** - * @author Soby Chacko - * @author Artem Bilan - * @author Marius Bogoevici - * @since 0.5 - */ -public class ProducerConfigurationTests { - - @Test - @SuppressWarnings("unchecked") - public void testSendMessageWithNonDefaultKeyAndValueEncoders() throws Exception { - final ProducerMetadata producerMetadata = new ProducerMetadata("test", String.class, String.class, new StringSerializer(), new StringSerializer()); - final Producer producer = Mockito.mock(Producer.class); - - final ProducerConfiguration configuration = - new ProducerConfiguration(producerMetadata, producer); - - configuration.send("test", "key", "test message"); - - Mockito.verify(producer, Mockito.times(1)).send(Mockito.any(ProducerRecord.class)); - - final ArgumentCaptor> argument = - (ArgumentCaptor>) (Object) - ArgumentCaptor.forClass(ProducerRecord.class); - Mockito.verify(producer).send(argument.capture()); - - final ProducerRecord capturedKeyMessage = argument.getValue(); - - Assert.assertEquals(capturedKeyMessage.key(), "key"); - Assert.assertEquals(capturedKeyMessage.value(), "test message"); - Assert.assertEquals(capturedKeyMessage.topic(), "test"); - } - - /** - * User does not set an explicit key/value encoder, but send a serializable object for both key/value - */ - @Test - @SuppressWarnings("unchecked") - public void testSendMessageWithDefaultKeyAndValueEncodersAndCustomSerializableKeyAndPayloadObject() - throws Exception { - final ProducerMetadata producerMetadata = new ProducerMetadata("test", - byte[].class,byte[].class, new ByteArraySerializer(), new ByteArraySerializer()); - final Producer producer = Mockito.mock(Producer.class); - - final ProducerConfiguration configuration = - new ProducerConfiguration(producerMetadata, producer); - - configuration.convertAndSend("test", new TestKey("compositePart1", "compositePart2"), new TestPayload("part1", "part2")); - - Mockito.verify(producer, Mockito.times(1)).send(Mockito.any(ProducerRecord.class)); - - final ArgumentCaptor> argument = - (ArgumentCaptor>) (Object) - ArgumentCaptor.forClass(ProducerRecord.class); - Mockito.verify(producer).send(argument.capture()); - - final ProducerRecord capturedKeyMessage = argument.getValue(); - - final byte[] keyBytes = capturedKeyMessage.key(); - - final ByteArrayInputStream keyInputStream = new ByteArrayInputStream(keyBytes); - final ObjectInputStream keyObjectInputStream = new ObjectInputStream(keyInputStream); - final Object keyObj = keyObjectInputStream.readObject(); - - final TestKey tk = (TestKey) keyObj; - - Assert.assertEquals(tk.getKeyPart1(), "compositePart1"); - Assert.assertEquals(tk.getKeyPart2(), "compositePart2"); - - final byte[] messageBytes = capturedKeyMessage.value(); - - final ByteArrayInputStream messageInputStream = new ByteArrayInputStream(messageBytes); - final ObjectInputStream messageObjectInputStream = new ObjectInputStream(messageInputStream); - final Object messageObj = messageObjectInputStream.readObject(); - - final TestPayload tp = (TestPayload) messageObj; - - Assert.assertEquals(tp.getPart1(), "part1"); - Assert.assertEquals(tp.getPart2(), "part2"); - - Assert.assertEquals(capturedKeyMessage.topic(), "test"); - } - - /** - * User does not set an explicit key encoder, but a value encoder, and sends the corresponding data - */ - @Test - @SuppressWarnings("unchecked") - public void testSendMessageWithDefaultKeyEncoderAndNonDefaultValueEncoderAndCorrespondingData() throws Exception { - final AvroReflectDatumBackedKafkaEncoder encoder = - new AvroReflectDatumBackedKafkaEncoder(TestPayload.class); - final ProducerMetadata producerMetadata = new ProducerMetadata("test", byte[].class,TestPayload.class, new ByteArraySerializer(), new EncoderAdaptingSerializer(encoder)); - - final Producer producer = Mockito.mock(Producer.class); - - final ProducerConfiguration configuration = - new ProducerConfiguration<>(producerMetadata, producer); - - TestPayload tp = new TestPayload("part1", "part2"); - configuration.convertAndSend("test", "key", tp); - - Mockito.verify(producer, Mockito.times(1)).send(Mockito.any(ProducerRecord.class)); - - final ArgumentCaptor> argument = - (ArgumentCaptor>) (Object) - ArgumentCaptor.forClass(ProducerRecord.class); - Mockito.verify(producer).send(argument.capture()); - - final ProducerRecord capturedKeyMessage = argument.getValue(); - - final byte[] keyBytes = capturedKeyMessage.key(); - - String keyObj = new String(keyBytes); - - Assert.assertEquals("key", keyObj); - Assert.assertEquals(capturedKeyMessage.value(), tp); - - Assert.assertEquals(capturedKeyMessage.topic(), "test"); - } - - /** - * User does set an explicit key encoder, but not a value encoder, and sends the corresponding data - */ - @Test - @SuppressWarnings("unchecked") - public void testSendMessageWithNonDefaultKeyEncoderAndDefaultValueEncoderAndCorrespondingData() throws Exception { - final AvroReflectDatumBackedKafkaEncoder encoder = new AvroReflectDatumBackedKafkaEncoder(TestKey.class); - final ProducerMetadata producerMetadata = new ProducerMetadata("test", TestKey.class, byte[].class, new EncoderAdaptingSerializer(encoder), new ByteArraySerializer()); - - final Producer producer = Mockito.mock(Producer.class); - - final ProducerConfiguration configuration = - new ProducerConfiguration(producerMetadata, producer); - - final TestKey tk = new TestKey("part1", "part2"); - - configuration.convertAndSend("test", tk, "test message"); - - Mockito.verify(producer, Mockito.times(1)).send(Mockito.any(ProducerRecord.class)); - - final ArgumentCaptor> argument = - (ArgumentCaptor>) (Object) - ArgumentCaptor.forClass(ProducerRecord.class); - Mockito.verify(producer).send(argument.capture()); - - final ProducerRecord capturedKeyMessage = argument.getValue(); - - Assert.assertEquals(capturedKeyMessage.key(), tk); - - final byte[] payloadBytes = capturedKeyMessage.value(); - - String payload = new String(payloadBytes); - Assert.assertEquals("test message", payload); - - Assert.assertEquals(capturedKeyMessage.topic(), "test"); - } - - /** - * User does not set an explicit key/value encoder, but send a serializable String key/value pair - */ - @Test - @SuppressWarnings("unchecked") - public void testSendMessageWithDefaultKeyAndValueEncodersAndStringKeyAndValue() throws Exception { - final ProducerMetadata producerMetadata = new ProducerMetadata("test", byte[].class, byte[].class, new ByteArraySerializer(), new ByteArraySerializer()); - final Producer producer = Mockito.mock(Producer.class); - - final ProducerConfiguration configuration = - new ProducerConfiguration(producerMetadata, producer); - - configuration.convertAndSend("test", "key", "test message"); - - Mockito.verify(producer, Mockito.times(1)).send(Mockito.any(ProducerRecord.class)); - - final ArgumentCaptor> argument = - (ArgumentCaptor>) (Object) - ArgumentCaptor.forClass(ProducerRecord.class); - Mockito.verify(producer).send(argument.capture()); - - final ProducerRecord capturedKeyMessage = argument.getValue(); - final byte[] keyBytes = capturedKeyMessage.key(); - - String key = new String(keyBytes); - Assert.assertEquals("key", key); - - final byte[] payloadBytes = capturedKeyMessage.value(); - - String payload = new String(payloadBytes); - Assert.assertEquals("test message", payload); - Assert.assertEquals(capturedKeyMessage.topic(), "test"); - } - - @Test - @SuppressWarnings("unchecked") - public void testSendMessageWithNonDefaultKeyAndValueEncodersSpecifyingPartition() throws Exception { - final ProducerMetadata producerMetadata = new ProducerMetadata("test", String.class, String.class, new StringSerializer(), new StringSerializer()); - final Producer producer = Mockito.mock(Producer.class); - - final ProducerConfiguration configuration = - new ProducerConfiguration(producerMetadata, producer); - - configuration.send("test", 1, "key", "test message"); - - Mockito.verify(producer, Mockito.times(1)).send(Mockito.any(ProducerRecord.class)); - - final ArgumentCaptor> argument = - (ArgumentCaptor>) (Object) - ArgumentCaptor.forClass(ProducerRecord.class); - Mockito.verify(producer).send(argument.capture()); - - final ProducerRecord capturedKeyMessage = argument.getValue(); - - Assert.assertEquals(capturedKeyMessage.key(), "key"); - Assert.assertEquals(capturedKeyMessage.value(), "test message"); - Assert.assertEquals(capturedKeyMessage.topic(), "test"); - Assert.assertEquals(capturedKeyMessage.partition().intValue(), 1); - } - - @Test - @SuppressWarnings("unchecked") - public void testSendMessageWithNonDefaultKeyAndValueEncodersCustomPartitioner() throws Exception { - Partitioner customPartitioner = Mockito.mock(Partitioner.class); - final ProducerMetadata producerMetadata = new ProducerMetadata("test", String.class, String.class, new StringSerializer(), new StringSerializer()); - producerMetadata.setPartitioner(customPartitioner); - final Producer producer = Mockito.mock(Producer.class); - - final ProducerConfiguration configuration = - new ProducerConfiguration(producerMetadata, producer); - - ArrayList partitionInfos = new ArrayList<>(); - partitionInfos.add(Mockito.mock(PartitionInfo.class)); - Mockito.when(producer.partitionsFor("test")).thenReturn(partitionInfos); - Mockito.when(customPartitioner.partition("key", 1)).thenReturn(4); - - configuration.send("test", null, "key", "test message"); - - Mockito.verify(producer, Mockito.times(1)).send(Mockito.any(ProducerRecord.class)); - - final ArgumentCaptor> argument = - (ArgumentCaptor>) (Object) - ArgumentCaptor.forClass(ProducerRecord.class); - Mockito.verify(producer).send(argument.capture()); - - final ProducerRecord capturedKeyMessage = argument.getValue(); - - Assert.assertEquals(capturedKeyMessage.key(), "key"); - Assert.assertEquals(capturedKeyMessage.value(), "test message"); - Assert.assertEquals(capturedKeyMessage.topic(), "test"); - Assert.assertEquals(capturedKeyMessage.partition().intValue(), 4); - } - - /** - * User does not set an explicit key/value encoder, but send non-serializable object for both key/value - */ - @Test(expected = ConversionFailedException.class) - @SuppressWarnings("unchecked") - public void testSendMessageWithDefaultKeyAndValueEncodersButNonSerializableKeyAndValue() throws Exception { - final ProducerMetadata producerMetadata = new ProducerMetadata("test", byte[].class, byte[].class, new ByteArraySerializer(), new ByteArraySerializer()); - final Producer producer = Mockito.mock(Producer.class); - - final ProducerConfiguration configuration = - new ProducerConfiguration(producerMetadata, producer); - - Message message = - new GenericMessage(new NonSerializableTestPayload("part1", "part2")); - - configuration.convertAndSend("test", new NonSerializableTestKey("compositePart1", "compositePart2"), message); - } - - /** - * User does not set an explicit key/value encoder, but send non-serializable key and serializable value - */ - @Test(expected = ConversionFailedException.class) - @SuppressWarnings("unchecked") - public void testSendMessageWithDefaultKeyAndValueEncodersButNonSerializableKeyAndSerializableValue() - throws Exception { - final ProducerMetadata producerMetadata = new ProducerMetadata("test", byte[].class, byte[].class, new ByteArraySerializer(), new ByteArraySerializer()); - final Producer producer = Mockito.mock(Producer.class); - - final ProducerConfiguration configuration = - new ProducerConfiguration(producerMetadata, producer); - - configuration.convertAndSend("test", new NonSerializableTestKey("compositePart1", "compositePart2"), new TestPayload("part1", "part2")); - } - - /** - * User does not set an explicit key/value encoder, but send serializable key and non-serializable value - */ - @Test(expected = ConversionFailedException.class) - @SuppressWarnings("unchecked") - public void testSendMessageWithDefaultKeyAndValueEncodersButSerializableKeyAndNonSerializableValue() - throws Exception { - final ProducerMetadata producerMetadata = new ProducerMetadata("test", byte[].class, byte[].class, new EncoderAdaptingSerializer(new DefaultEncoder(null)), new EncoderAdaptingSerializer(new DefaultEncoder(null))); - final Producer producer = Mockito.mock(Producer.class); - - final ProducerConfiguration configuration = - new ProducerConfiguration(producerMetadata, producer); - - configuration.convertAndSend("test", new TestKey("compositePart1", "compositePart2"), new NonSerializableTestPayload("part1", "part2")); - } - -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/support/ProducerFactoryBeanTests.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/support/ProducerFactoryBeanTests.java deleted file mode 100644 index 439e956c96..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/support/ProducerFactoryBeanTests.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.kafka.support; - -import org.apache.kafka.clients.producer.Producer; -import org.apache.kafka.common.serialization.ByteArraySerializer; -import org.junit.Assert; -import org.junit.Test; -import org.mockito.Mockito; - -/** - * @author Soby Chacko - * @since 0.5 - */ -public class ProducerFactoryBeanTests { - - @Test - public void createProducerWithDefaultMetadata() throws Exception { - final ProducerMetadata producerMetadata = new ProducerMetadata("test", byte[].class, byte[].class, new ByteArraySerializer(), new ByteArraySerializer()); - final ProducerMetadata tm = Mockito.spy(producerMetadata); - final ProducerFactoryBean producerFactoryBean = new ProducerFactoryBean(tm, "localhost:9092"); - final Producer producer = producerFactoryBean.getObject(); - - Assert.assertTrue(producer != null); - - Mockito.verify(tm, Mockito.times(1)).getCompressionType(); - Mockito.verify(tm, Mockito.times(1)).getValueSerializer(); - Mockito.verify(tm, Mockito.times(1)).getKeySerializer(); - } -} diff --git a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/test/utils/User.java b/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/test/utils/User.java deleted file mode 100644 index 8ceada801e..0000000000 --- a/spring-integration-kafka/src/test/java/org/springframework/integration/kafka/test/utils/User.java +++ /dev/null @@ -1,93 +0,0 @@ -package org.springframework.integration.kafka.test.utils; - -import org.apache.avro.specific.SpecificRecord; - -/** - * @author Soby Chacko - * @since 0.5 - *

- * This class is copied (partly) from an Avro generated class for necessary testing. - * Please use caution when modify. - */ -public class User extends org.apache.avro.specific.SpecificRecordBase implements SpecificRecord { - - public static final org.apache.avro.Schema SCHEMA$ = new org.apache.avro.Schema.Parser().parse("{\"type\":\"record\",\"name\":\"User\",\"namespace\":\"org.springframework.integration.samples.kafka.user\",\"fields\":[{\"name\":\"firstName\",\"type\":\"string\"},{\"name\":\"lastName\",\"type\":\"string\"}]}"); - public java.lang.CharSequence firstName; - public java.lang.CharSequence lastName; - - /** - * Default constructor. - */ - public User() { - } - - /** - * All-args constructor. - */ - public User(java.lang.CharSequence firstName, java.lang.CharSequence lastName) { - this.firstName = firstName; - this.lastName = lastName; - } - - public org.apache.avro.Schema getSchema() { - return SCHEMA$; - } - - // Used by DatumWriter. Applications should not call. - public java.lang.Object get(int field$) { - switch (field$) { - case 0: - return firstName; - case 1: - return lastName; - default: - throw new org.apache.avro.AvroRuntimeException("Bad index"); - } - } - - // Used by DatumReader. Applications should not call. - public void put(int field$, java.lang.Object value$) { - switch (field$) { - case 0: - firstName = (java.lang.CharSequence) value$; - break; - case 1: - lastName = (java.lang.CharSequence) value$; - break; - default: - throw new org.apache.avro.AvroRuntimeException("Bad index"); - } - } - - /** - * Gets the value of the 'firstName' field. - */ - public java.lang.CharSequence getFirstName() { - return firstName; - } - - /** - * Sets the value of the 'firstName' field. - * - * @param value the value to set. - */ - public void setFirstName(java.lang.CharSequence value) { - this.firstName = value; - } - - /** - * Gets the value of the 'lastName' field. - */ - public java.lang.CharSequence getLastName() { - return lastName; - } - - /** - * Sets the value of the 'lastName' field. - * - * @param value the value to set. - */ - public void setLastName(java.lang.CharSequence value) { - this.lastName = value; - } -} diff --git a/spring-integration-kafka/src/test/resources/log4j.properties b/spring-integration-kafka/src/test/resources/log4j.properties deleted file mode 100644 index f5f5f3efe7..0000000000 --- a/spring-integration-kafka/src/test/resources/log4j.properties +++ /dev/null @@ -1,10 +0,0 @@ -log4j.rootCategory=WARN, stdout - -log4j.appender.stdout=org.apache.log4j.ConsoleAppender -log4j.appender.stdout.layout=org.apache.log4j.PatternLayout -log4j.appender.stdout.layout.ConversionPattern=%d{HH:mm:ss.SSS} %-5p [%t][%c] %m%n - -log4j.category.org.springframework.integration=WARN -log4j.category.org.springframework.integration.kafka=INFO -log4j.category.org.apache.kafka.common.network.Selector=ERROR -log4j.category.kafka.server.ReplicaFetcherThread=ERROR