From 5377eaf258f6f06b0798d4d8554220ea8b0f38be Mon Sep 17 00:00:00 2001 From: David Turanski Date: Thu, 9 Jul 2015 10:47:09 -0400 Subject: [PATCH] XD-3242 created spring-cloud-streams-codec Port MessageBus code from XD (XD-3244) Fix Deprecated API Usage; Fix Imports (GH-9) Change poms to compile using JDK7 Move transport specific test classes into binding- projects --- docs/pom.xml | 1 - pom.xml | 27 +- spring-cloud-streams-bindings/pom.xml | 95 ++ .../pom.xml | 91 ++ .../kafka/IntegerEncoderDecoder.java | 58 + .../integration/kafka/KafkaMessageBus.java | 947 ++++++++++++++ .../kafka/WindowingOffsetManager.java | 263 ++++ .../META-INF/spring-xd/bus/kafka-bus.xml | 49 + .../bus/kafka/KafkaMessageBusTests.java | 315 +++++ .../bus/kafka/KafkaTestMessageBus.java | 76 ++ .../kafka/RawKafkaPartitionTestSupport.java | 40 + .../kafka/RawModeKafkaMessageBusTests.java | 337 +++++ .../integration/kafka/EmbeddedZookeeper.java | 108 ++ .../integration/kafka/KafkaTestSupport.java | 167 +++ .../integration/kafka/TestKafkaCluster.java | 171 +++ .../pom.xml | 31 + .../bus/local/LocalMessageBus.java | 418 ++++++ .../META-INF/spring-xd/bus/local-bus.xml | 18 + .../bus/local/LocalMessageBusTests.java | 184 +++ .../pom.xml | 40 + .../rabbit/ConnectionFactorySettings.java | 84 ++ .../LocalizedQueueConnectionFactory.java | 232 ++++ .../integration/rabbit/RabbitBusCleaner.java | 256 ++++ .../integration/rabbit/RabbitMessageBus.java | 1066 ++++++++++++++++ .../dirt/integration/rabbit/package-info.java | 5 + .../META-INF/spring-xd/bus/rabbit-bus.xml | 56 + ...ueueConnectionFactoryIntegrationTests.java | 69 + .../LocalizedQueueConnectionFactoryTests.java | 186 +++ .../rabbit/RabbitAdminTestSupport.java | 57 + .../rabbit/RabbitBusCleanerTests.java | 216 ++++ .../rabbit/RabbitMessageBusTests.java | 715 +++++++++++ .../rabbit/RabbitTestMessageBus.java | 77 ++ .../integration/rabbit/RabbitTestSupport.java | 61 + .../src/test/resources/log4j.properties | 8 + .../pom.xml | 58 + .../integration/redis/RedisMessageBus.java | 532 ++++++++ .../dirt/integration/redis/package-info.java | 5 + .../META-INF/spring-xd/bus/redis-bus.xml | 19 + .../bus/redis/RedisMessageBusTests.java | 395 ++++++ .../bus/redis/RedisTestMessageBus.java | 73 ++ .../redis/AbstractRedisSerializerTests.java | 153 +++ .../RedisPublishingMessageHandlerTests.java | 144 +++ .../RedisQueueInboundChannelAdapterTests.java | 216 ++++ ...RedisQueueOutboundChannelAdapterTests.java | 171 +++ .../integration/redis/RedisTestSupport.java | 47 + .../spring-cloud-streams-binding-spi/pom.xml | 47 + .../bus/AbstractBusPropertiesAccessor.java | 379 ++++++ .../xd/dirt/integration/bus/Binding.java | 119 ++ .../xd/dirt/integration/bus/BusCleaner.java | 39 + .../dirt/integration/bus/BusProperties.java | 144 +++ .../xd/dirt/integration/bus/BusUtils.java | 90 ++ .../bus/EmbeddedHeadersMessageConverter.java | 158 +++ .../xd/dirt/integration/bus/MessageBus.java | 158 +++ .../integration/bus/MessageBusException.java | 33 + .../integration/bus/MessageBusSupport.java | 1133 +++++++++++++++++ .../dirt/integration/bus/MessageValues.java | 155 +++ .../bus/PartitionKeyExtractorStrategy.java | 31 + .../bus/PartitionSelectorStrategy.java | 41 + .../integration/bus/RabbitAdminException.java | 39 + .../bus/RabbitManagementUtils.java | 82 ++ .../StringConvertingContentTypeResolver.java | 57 + .../xd/dirt/integration/bus/XdHeaders.java | 62 + .../bus/MessageConverterTests.java | 96 ++ .../README.md | 2 + .../spring-cloud-streams-binding-test/pom.xml | 120 ++ .../bus/AbstractMessageBusTests.java | 311 +++++ .../bus/AbstractTestMessageBus.java | 141 ++ .../dirt/integration/bus/BrokerBusTests.java | 114 ++ .../xd/dirt/integration/bus/BusTestUtils.java | 49 + .../bus/PartitionCapableBusTests.java | 271 ++++ .../integration/bus/PartitionTestSupport.java | 38 + .../xd/dirt/integration/bus/Spy.java | 13 + .../AbstractExternalResourceTestSupport.java | 134 ++ .../springframework/xd/test/TestUtils.java | 60 + .../bus/MessageBusSupportTests.java | 302 +++++ spring-cloud-streams-codec/pom.xml | 4 - spring-cloud-streams-common/pom.xml | 4 - spring-cloud-streams-samples/double/pom.xml | 6 +- spring-cloud-streams-samples/extended/pom.xml | 7 +- spring-cloud-streams-samples/pom.xml | 9 +- spring-cloud-streams-samples/sink/pom.xml | 7 +- spring-cloud-streams-samples/source/pom.xml | 7 +- spring-cloud-streams-samples/tap/pom.xml | 7 +- .../transform/pom.xml | 6 +- spring-cloud-streams/pom.xml | 19 +- .../ChannelBindingAdapterConfiguration.java | 13 +- .../streams/config/ModulePostProcessor.java | 4 +- .../MessageBusAwareChannelResolverTests.java | 170 +++ spring-xd-runner/pom.xml | 4 - 89 files changed, 12948 insertions(+), 74 deletions(-) create mode 100644 spring-cloud-streams-bindings/pom.xml create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-kafka/pom.xml create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-kafka/src/main/java/org/springframework/xd/dirt/integration/kafka/IntegerEncoderDecoder.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-kafka/src/main/java/org/springframework/xd/dirt/integration/kafka/KafkaMessageBus.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-kafka/src/main/java/org/springframework/xd/dirt/integration/kafka/WindowingOffsetManager.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-kafka/src/main/resources/META-INF/spring-xd/bus/kafka-bus.xml create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-kafka/src/test/java/org/springframework/xd/dirt/integration/bus/kafka/KafkaMessageBusTests.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-kafka/src/test/java/org/springframework/xd/dirt/integration/bus/kafka/KafkaTestMessageBus.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-kafka/src/test/java/org/springframework/xd/dirt/integration/bus/kafka/RawKafkaPartitionTestSupport.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-kafka/src/test/java/org/springframework/xd/dirt/integration/bus/kafka/RawModeKafkaMessageBusTests.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-kafka/src/test/java/org/springframework/xd/dirt/integration/kafka/EmbeddedZookeeper.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-kafka/src/test/java/org/springframework/xd/dirt/integration/kafka/KafkaTestSupport.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-kafka/src/test/java/org/springframework/xd/dirt/integration/kafka/TestKafkaCluster.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-local/pom.xml create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-local/src/main/java/org/springframework/xd/dirt/integration/bus/local/LocalMessageBus.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-local/src/main/resources/META-INF/spring-xd/bus/local-bus.xml create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-local/src/test/java/org/springframework/xd/dirt/integration/bus/local/LocalMessageBusTests.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/pom.xml create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/main/java/org/springframework/xd/dirt/integration/rabbit/ConnectionFactorySettings.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/main/java/org/springframework/xd/dirt/integration/rabbit/LocalizedQueueConnectionFactory.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/main/java/org/springframework/xd/dirt/integration/rabbit/RabbitBusCleaner.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/main/java/org/springframework/xd/dirt/integration/rabbit/RabbitMessageBus.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/main/java/org/springframework/xd/dirt/integration/rabbit/package-info.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/main/resources/META-INF/spring-xd/bus/rabbit-bus.xml create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/test/java/org/springframework/xd/dirt/integration/rabbit/LocalizedQueueConnectionFactoryIntegrationTests.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/test/java/org/springframework/xd/dirt/integration/rabbit/LocalizedQueueConnectionFactoryTests.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/test/java/org/springframework/xd/dirt/integration/rabbit/RabbitAdminTestSupport.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/test/java/org/springframework/xd/dirt/integration/rabbit/RabbitBusCleanerTests.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/test/java/org/springframework/xd/dirt/integration/rabbit/RabbitMessageBusTests.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/test/java/org/springframework/xd/dirt/integration/rabbit/RabbitTestMessageBus.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/test/java/org/springframework/xd/dirt/integration/rabbit/RabbitTestSupport.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/test/resources/log4j.properties create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-redis/pom.xml create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-redis/src/main/java/org/springframework/xd/dirt/integration/redis/RedisMessageBus.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-redis/src/main/java/org/springframework/xd/dirt/integration/redis/package-info.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-redis/src/main/resources/META-INF/spring-xd/bus/redis-bus.xml create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-redis/src/test/java/org/springframework/xd/dirt/integration/bus/redis/RedisMessageBusTests.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-redis/src/test/java/org/springframework/xd/dirt/integration/bus/redis/RedisTestMessageBus.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-redis/src/test/java/org/springframework/xd/dirt/integration/redis/AbstractRedisSerializerTests.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-redis/src/test/java/org/springframework/xd/dirt/integration/redis/RedisPublishingMessageHandlerTests.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-redis/src/test/java/org/springframework/xd/dirt/integration/redis/RedisQueueInboundChannelAdapterTests.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-redis/src/test/java/org/springframework/xd/dirt/integration/redis/RedisQueueOutboundChannelAdapterTests.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-redis/src/test/java/org/springframework/xd/dirt/integration/redis/RedisTestSupport.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/pom.xml create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/AbstractBusPropertiesAccessor.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/Binding.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/BusCleaner.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/BusProperties.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/BusUtils.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/EmbeddedHeadersMessageConverter.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/MessageBus.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/MessageBusException.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/MessageBusSupport.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/MessageValues.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/PartitionKeyExtractorStrategy.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/PartitionSelectorStrategy.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/RabbitAdminException.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/RabbitManagementUtils.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/StringConvertingContentTypeResolver.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/XdHeaders.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/test/java/org/springframework/xd/dirt/integration/bus/MessageConverterTests.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-test/README.md create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-test/pom.xml create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-test/src/main/java/org/springframework/xd/dirt/integration/bus/AbstractMessageBusTests.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-test/src/main/java/org/springframework/xd/dirt/integration/bus/AbstractTestMessageBus.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-test/src/main/java/org/springframework/xd/dirt/integration/bus/BrokerBusTests.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-test/src/main/java/org/springframework/xd/dirt/integration/bus/BusTestUtils.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-test/src/main/java/org/springframework/xd/dirt/integration/bus/PartitionCapableBusTests.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-test/src/main/java/org/springframework/xd/dirt/integration/bus/PartitionTestSupport.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-test/src/main/java/org/springframework/xd/dirt/integration/bus/Spy.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-test/src/main/java/org/springframework/xd/test/AbstractExternalResourceTestSupport.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-test/src/main/java/org/springframework/xd/test/TestUtils.java create mode 100644 spring-cloud-streams-bindings/spring-cloud-streams-binding-test/src/test/java/org/springframework/xd/dirt/integration/bus/MessageBusSupportTests.java create mode 100644 spring-cloud-streams/src/test/java/org/springframework/xd/dirt/integration/bus/MessageBusAwareChannelResolverTests.java diff --git a/docs/pom.xml b/docs/pom.xml index b79868017..4445e3ab1 100644 --- a/docs/pom.xml +++ b/docs/pom.xml @@ -10,7 +10,6 @@ spring-cloud-streams-docs pom Spring Cloud Streams Docs - 1.1.0.BUILD-SNAPSHOT Spring Cloud Docs spring-cloud-streams diff --git a/pom.xml b/pom.xml index 2c1d4347b..c2d69e089 100644 --- a/pom.xml +++ b/pom.xml @@ -27,6 +27,7 @@ spring-cloud-streams + spring-cloud-streams-bindings spring-cloud-streams-codec spring-cloud-streams-common spring-xd-runner @@ -75,19 +76,19 @@ ${spring-xd.version} - org.springframework.xd - spring-xd-messagebus-local - ${spring-xd.version} + org.springframework.cloud + spring-cloud-streams-binding-local + ${project.version} - org.springframework.xd - spring-xd-messagebus-redis - ${spring-xd.version} + org.springframework.cloud + spring-cloud-streams-binding-redis + ${project.version} - org.springframework.xd - spring-xd-messagebus-rabbit - ${spring-xd.version} + org.springframework.cloud + spring-cloud-streams-binding-rabbit + ${project.version} org.springframework @@ -118,6 +119,14 @@ false + + jcenter + JCenter Bintray + http://jcenter.bintray.com + + false + + diff --git a/spring-cloud-streams-bindings/pom.xml b/spring-cloud-streams-bindings/pom.xml new file mode 100644 index 000000000..95a0efbce --- /dev/null +++ b/spring-cloud-streams-bindings/pom.xml @@ -0,0 +1,95 @@ + + + 4.0.0 + + spring-cloud-streams-bindings-parent + pom + http://projects.spring.io/spring-xd/ + + Pivotal Software, Inc. + http://www.spring.io + + + org.springframework.cloud + spring-cloud-streams-parent + 1.0.0.BUILD-SNAPSHOT + + + 0.8.2.1 + 2.6.0 + + + spring-cloud-streams-binding-spi + spring-cloud-streams-binding-test + spring-cloud-streams-binding-local + spring-cloud-streams-binding-rabbit + spring-cloud-streams-binding-redis + spring-cloud-streams-binding-kafka + + + + + + org.springframework.cloud + spring-cloud-streams-binding-spi + ${project.version} + + + org.springframework.cloud + spring-cloud-streams-binding-test + ${project.version} + test + + + org.apache.kafka + kafka_2.10 + ${kafka.version} + + + org.slf4j + slf4j-log4j12 + + + + + org.apache.kafka + kafka_2.10 + test + ${kafka.version} + + + org.apache.kafka + kafka-clients + ${kafka.version} + + + org.apache.curator + curator-framework + ${curator.version} + + + org.apache.curator + curator-recipes + ${curator.version} + + + org.apache.curator + curator-test + ${curator.version} + + + + + + + org.springframework.boot + spring-boot-starter-logging + + + org.springframework.boot + spring-boot-starter-test + test + + + diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-kafka/pom.xml b/spring-cloud-streams-bindings/spring-cloud-streams-binding-kafka/pom.xml new file mode 100644 index 000000000..6ce0ea980 --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-kafka/pom.xml @@ -0,0 +1,91 @@ + + + 4.0.0 + + spring-cloud-streams-binding-kafka + jar + spring-cloud-streams-binding-kafka + Kafka binding implementation + + + org.springframework.cloud + spring-cloud-streams-bindings-parent + 1.0.0.BUILD-SNAPSHOT + + + + UTF-8 + 0.8.2.1 + 1.2.0.RELEASE + 1.0.0 + + + + + org.springframework.cloud + spring-cloud-streams-binding-spi + + + org.springframework.cloud + spring-cloud-streams-binding-test + + + org.springframework.integration + spring-integration-kafka + ${spring-integration-kafka.version} + + + org.apache.avro + avro-compiler + + + + + org.apache.kafka + kafka_2.10 + + + org.apache.kafka + kafka-clients + + + io.reactivex + rxjava + ${rxjava.version} + + + io.reactivex + rxjava-math + ${rxjava.version} + + + org.springframework.xd + spring-xd-tuple + ${spring-xd.version} + + + org.springframework.xd + spring-xd-codec + + + test + + + org.apache.curator + curator-recipes + test + + + org.apache.kafka + kafka_2.10 + test + test + + + org.apache.curator + curator-test + test + + + diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-kafka/src/main/java/org/springframework/xd/dirt/integration/kafka/IntegerEncoderDecoder.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-kafka/src/main/java/org/springframework/xd/dirt/integration/kafka/IntegerEncoderDecoder.java new file mode 100644 index 000000000..2b7b6501f --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-kafka/src/main/java/org/springframework/xd/dirt/integration/kafka/IntegerEncoderDecoder.java @@ -0,0 +1,58 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.xd.dirt.integration.kafka; + +import kafka.serializer.Decoder; +import kafka.serializer.Encoder; +import kafka.utils.VerifiableProperties; + +import org.springframework.util.Assert; + + +/** + * A Kafka encoder / decoder used to serialize a single int, used as the kafka partition key. + * + * @author Eric Bottard + */ +public class IntegerEncoderDecoder implements Encoder, Decoder { + + + public IntegerEncoderDecoder() { + this(new VerifiableProperties()); + } + + public IntegerEncoderDecoder(VerifiableProperties properties) { + } + + @Override + public Integer fromBytes(byte[] bytes) { + Assert.isTrue(bytes.length == 4); + return bytes[0] << 24 | (bytes[1] & 0xFF) << 16 | (bytes[2] & 0xFF) << 8 | (bytes[3] & 0xFF); + } + + @Override + public byte[] toBytes(Integer message) { + int value = message.intValue(); + return new byte[] { + (byte) (value >>> 24), + (byte) (value >>> 16), + (byte) (value >>> 8), + (byte) value + }; + } + +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-kafka/src/main/java/org/springframework/xd/dirt/integration/kafka/KafkaMessageBus.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-kafka/src/main/java/org/springframework/xd/dirt/integration/kafka/KafkaMessageBus.java new file mode 100644 index 000000000..e6c5244bd --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-kafka/src/main/java/org/springframework/xd/dirt/integration/kafka/KafkaMessageBus.java @@ -0,0 +1,947 @@ +/* + * Copyright 2014-2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.xd.dirt.integration.kafka; + +import java.io.UnsupportedEncodingException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.UUID; +import java.util.concurrent.atomic.AtomicInteger; + +import kafka.admin.AdminUtils; +import kafka.api.OffsetRequest; +import kafka.serializer.Decoder; +import kafka.serializer.DefaultDecoder; +import kafka.utils.ZkUtils; + +import org.I0Itec.zkclient.ZkClient; +import org.I0Itec.zkclient.exception.ZkMarshallingError; +import org.I0Itec.zkclient.serialize.ZkSerializer; +import org.apache.kafka.clients.producer.ProducerConfig; +import org.apache.kafka.common.serialization.ByteArraySerializer; + +import org.springframework.beans.factory.DisposableBean; +import org.springframework.http.MediaType; +import org.springframework.integration.channel.FixedSubscriberChannel; +import org.springframework.integration.endpoint.EventDrivenConsumer; +import org.springframework.integration.handler.AbstractMessageHandler; +import org.springframework.integration.handler.AbstractReplyProducingMessageHandler; +import org.springframework.integration.kafka.core.ConnectionFactory; +import org.springframework.integration.kafka.core.DefaultConnectionFactory; +import org.springframework.integration.kafka.core.Partition; +import org.springframework.integration.kafka.core.ZookeeperConfiguration; +import org.springframework.integration.kafka.inbound.KafkaMessageDrivenChannelAdapter; +import org.springframework.integration.kafka.listener.Acknowledgment; +import org.springframework.integration.kafka.listener.KafkaMessageListenerContainer; +import org.springframework.integration.kafka.listener.KafkaTopicOffsetManager; +import org.springframework.integration.kafka.listener.OffsetManager; +import org.springframework.integration.kafka.support.KafkaHeaders; +import org.springframework.integration.kafka.support.ProducerConfiguration; +import org.springframework.integration.kafka.support.ProducerFactoryBean; +import org.springframework.integration.kafka.support.ProducerMetadata; +import org.springframework.integration.kafka.support.ZookeeperConnect; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageChannel; +import org.springframework.messaging.MessageHandler; +import org.springframework.messaging.MessageHeaders; +import org.springframework.messaging.SubscribableChannel; +import org.springframework.messaging.support.MessageBuilder; +import org.springframework.retry.RetryCallback; +import org.springframework.retry.RetryContext; +import org.springframework.retry.RetryOperations; +import org.springframework.retry.backoff.ExponentialBackOffPolicy; +import org.springframework.retry.policy.SimpleRetryPolicy; +import org.springframework.retry.support.RetryTemplate; +import org.springframework.util.Assert; +import org.springframework.util.CollectionUtils; +import org.springframework.util.StringUtils; +import org.springframework.xd.dirt.integration.bus.AbstractBusPropertiesAccessor; +import org.springframework.xd.dirt.integration.bus.Binding; +import org.springframework.xd.dirt.integration.bus.BusProperties; +import org.springframework.xd.dirt.integration.bus.EmbeddedHeadersMessageConverter; +import org.springframework.xd.dirt.integration.bus.MessageBusSupport; +import org.springframework.xd.dirt.integration.bus.MessageValues; +import org.springframework.xd.dirt.integration.bus.XdHeaders; +import org.springframework.xd.dirt.integration.bus.serializer.MultiTypeCodec; + +import scala.collection.Seq; + +/** + * A message bus that uses Kafka as the underlying middleware. The general implementation mapping between XD concepts + * and Kafka concepts is as follows: + * A message bus that uses Kafka as the underlying middleware. + * The general implementation mapping between XD concepts and Kafka concepts is as follows: + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + * + *
Stream definitionKafka topicKafka partitionsNotes
foo = "http | log"foo.01 partition1 producer, 1 consumer
foo = "http | log", log.count=xfoo.0x partitions1 producer, x consumers with static + * group 'springXD', achieves queue semantics
foo = "http | log", log.count=x + XD partitioningstill 1 topic 'foo.0'x partitions + use key + * computed by XD1 producer, x consumers with static group 'springXD', achieves queue semantics
foo = "http | log", log.count=x, concurrency=yfoo.0x*y partitions1 producer, x XD + * consumers, each with y threads
foo = "http | log", log.count=0, x actual log containersfoo.010(configurable) + * partitions1 producer, x XD consumers. Can't know the number of partitions beforehand, so decide a number + * that better be greater than number of containers
+ * @author Eric Bottard + * @author Marius Bogoevici + * @author Ilayaperumal Gopinathan + * @author David Turanski + * @author Gary Russell + */ +public class KafkaMessageBus extends MessageBusSupport { + + public static final ByteArraySerializer BYTE_ARRAY_SERIALIZER = new ByteArraySerializer(); + + public static final int METADATA_VERIFICATION_RETRY_ATTEMPTS = 10; + + public static final double METADATA_VERIFICATION_RETRY_BACKOFF_MULTIPLIER = 2; + + public static final int METADATA_VERIFICATION_RETRY_INITIAL_INTERVAL = 100; + + public static final int METADATA_VERIFICATION_MAX_INTERVAL = 1000; + + public static final String FETCH_SIZE = "fetchSize"; + + public static final String QUEUE_SIZE = "fetchSize"; + + public static final String REQUIRED_ACKS = "requiredAcks"; + + public static final String COMPRESSION_CODEC = "compressionCodec"; + + public static final String AUTO_COMMIT_ENABLED = "autoCommitEnabled"; + + private static final String DEFAULT_COMPRESSION_CODEC = "none"; + + private static final int DEFAULT_REQUIRED_ACKS = 1; + + private static final boolean DEFAULT_AUTO_COMMIT_ENABLED = true; + + private RetryOperations retryOperations; + + /** + * Used when writing directly to ZK. This is what Kafka expects. + */ + public final static ZkSerializer utf8Serializer = new ZkSerializer() { + + @Override + public byte[] serialize(Object data) throws ZkMarshallingError { + try { + return ((String) data).getBytes("UTF-8"); + } + catch (UnsupportedEncodingException e) { + throw new ZkMarshallingError(e); + } + } + + @Override + public Object deserialize(byte[] bytes) throws ZkMarshallingError { + try { + return new String(bytes, "UTF-8"); + } + catch (UnsupportedEncodingException e) { + throw new ZkMarshallingError(e); + } + } + }; + + protected static final Set PRODUCER_COMPRESSION_PROPERTIES = new HashSet( + Arrays.asList(new String[] { + KafkaMessageBus.COMPRESSION_CODEC, + })); + + /** + * The consumer group to use when achieving point to point semantics (that consumer group name is static and hence + * shared by all containers). + */ + private static final String POINT_TO_POINT_SEMANTICS_CONSUMER_GROUP = "springXD"; + + private static final Set KAFKA_CONSUMER_PROPERTIES = new SetBuilder() + .add(BusProperties.MIN_PARTITION_COUNT) + .build(); + + /** + * Basic + concurrency + partitioning. + */ + private static final Set SUPPORTED_CONSUMER_PROPERTIES = new SetBuilder() + .addAll(CONSUMER_STANDARD_PROPERTIES) + .addAll(KAFKA_CONSUMER_PROPERTIES) + .add(BusProperties.PARTITION_INDEX) // Not actually used + .add(BusProperties.COUNT) // Not actually used + .add(BusProperties.CONCURRENCY) + .add(FETCH_SIZE) + .build(); + + private static final Set KAFKA_PRODUCER_PROPERTIES = new SetBuilder() + .add(BusProperties.MIN_PARTITION_COUNT) + .build(); + + /** + * Basic + concurrency. + */ + private static final Set SUPPORTED_NAMED_CONSUMER_PROPERTIES = new SetBuilder() + .addAll(CONSUMER_STANDARD_PROPERTIES) + .build(); + + private static final Set SUPPORTED_NAMED_PRODUCER_PROPERTIES = new SetBuilder() + .addAll(PRODUCER_STANDARD_PROPERTIES) + .addAll(PRODUCER_BATCHING_BASIC_PROPERTIES) + .build(); + + /** + * Partitioning + kafka producer properties. + */ + private static final Set SUPPORTED_PRODUCER_PROPERTIES = new SetBuilder() + .addAll(PRODUCER_PARTITIONING_PROPERTIES) + .addAll(PRODUCER_STANDARD_PROPERTIES) + .add(BusProperties.DIRECT_BINDING_ALLOWED) + .addAll(KAFKA_PRODUCER_PROPERTIES) + .addAll(PRODUCER_BATCHING_BASIC_PROPERTIES) + .addAll(PRODUCER_COMPRESSION_PROPERTIES) + .build(); + + private final EmbeddedHeadersMessageConverter embeddedHeadersMessageConverter = new + EmbeddedHeadersMessageConverter(); + + private final ZookeeperConnect zookeeperConnect; + + private String brokers; + + private String[] headersToMap; + + private String zkAddress; + + // -------- Default values for properties ------- + private int defaultReplicationFactor = 1; + + private String defaultCompressionCodec = DEFAULT_COMPRESSION_CODEC; + + private int defaultRequiredAcks = DEFAULT_REQUIRED_ACKS; + + private int defaultQueueSize = 1024; + + private int defaultMaxWait = 100; + + private int defaultFetchSize = 1024 * 1024; + + private int defaultMinPartitionCount = 1; + + private ConnectionFactory connectionFactory; + + private String offsetStoreTopic = "SpringXdOffsets"; + + // auto commit property + + private boolean defaultAutoCommitEnabled = DEFAULT_AUTO_COMMIT_ENABLED; + + private int socketBufferSize = 2097152; + + private int offsetStoreSegmentSize = 250 * 1024 * 1024; + + private int offsetStoreRetentionTime = 60000; + + private int offsetStoreRequiredAcks = 1; + + private int offsetStoreMaxFetchSize = 1048576; + + private int offsetStoreBatchBytes = 200; + + private int offsetStoreBatchTime = 1000; + + private int offsetUpdateTimeWindow = 10000; + + private int offsetUpdateCount = 0; + + private int offsetUpdateShutdownTimeout = 2000; + + private Mode mode = Mode.embeddedHeaders; + + public KafkaMessageBus(ZookeeperConnect zookeeperConnect, String brokers, String zkAddress, + MultiTypeCodec codec, String... headersToMap) { + this.zookeeperConnect = zookeeperConnect; + this.brokers = brokers; + this.zkAddress = zkAddress; + setCodec(codec); + if (headersToMap.length > 0) { + String[] combinedHeadersToMap = + Arrays.copyOfRange(XdHeaders.STANDARD_HEADERS, 0, XdHeaders.STANDARD_HEADERS.length + headersToMap + .length); + System.arraycopy(headersToMap, 0, combinedHeadersToMap, XdHeaders.STANDARD_HEADERS.length, headersToMap + .length); + this.headersToMap = combinedHeadersToMap; + } + else { + this.headersToMap = XdHeaders.STANDARD_HEADERS; + } + + } + + public void setOffsetStoreTopic(String offsetStoreTopic) { + this.offsetStoreTopic = offsetStoreTopic; + } + + public void setOffsetStoreSegmentSize(int offsetStoreSegmentSize) { + this.offsetStoreSegmentSize = offsetStoreSegmentSize; + } + + public void setOffsetStoreRetentionTime(int offsetStoreRetentionTime) { + this.offsetStoreRetentionTime = offsetStoreRetentionTime; + } + + public void setSocketBufferSize(int socketBufferSize) { + this.socketBufferSize = socketBufferSize; + } + + public void setOffsetStoreRequiredAcks(int offsetStoreRequiredAcks) { + this.offsetStoreRequiredAcks = offsetStoreRequiredAcks; + } + + public void setOffsetStoreMaxFetchSize(int offsetStoreMaxFetchSize) { + this.offsetStoreMaxFetchSize = offsetStoreMaxFetchSize; + } + + + public void setOffsetUpdateTimeWindow(int offsetUpdateTimeWindow) { + this.offsetUpdateTimeWindow = offsetUpdateTimeWindow; + } + + public void setOffsetUpdateCount(int offsetUpdateCount) { + this.offsetUpdateCount = offsetUpdateCount; + } + + public void setOffsetUpdateShutdownTimeout(int offsetUpdateShutdownTimeout) { + this.offsetUpdateShutdownTimeout = offsetUpdateShutdownTimeout; + } + + public void setOffsetStoreBatchBytes(int offsetStoreBatchBytes) { + this.offsetStoreBatchBytes = offsetStoreBatchBytes; + } + + public void setOffsetStoreBatchTime(int offsetStoreBatchTime) { + this.offsetStoreBatchTime = offsetStoreBatchTime; + } + + public ConnectionFactory getConnectionFactory() { + return connectionFactory; + } + + /** + * Retry configuration for operations such as validating topic creation + * @param retryOperations the retry configuration + */ + public void setRetryOperations(RetryOperations retryOperations) { + this.retryOperations = retryOperations; + } + + @Override + public void afterPropertiesSet() throws Exception { + // we instantiate the connection factory here due to https://jira.spring.io/browse/XD-2647 + ZookeeperConfiguration configuration = new ZookeeperConfiguration(this.zookeeperConnect); + configuration.setBufferSize(socketBufferSize); + configuration.setMaxWait(defaultMaxWait); + DefaultConnectionFactory defaultConnectionFactory = + new DefaultConnectionFactory(configuration); + defaultConnectionFactory.afterPropertiesSet(); + this.connectionFactory = defaultConnectionFactory; + if (retryOperations == null) { + RetryTemplate retryTemplate = new RetryTemplate(); + + SimpleRetryPolicy simpleRetryPolicy = new SimpleRetryPolicy(); + simpleRetryPolicy.setMaxAttempts(METADATA_VERIFICATION_RETRY_ATTEMPTS); + retryTemplate.setRetryPolicy(simpleRetryPolicy); + + ExponentialBackOffPolicy backOffPolicy = new ExponentialBackOffPolicy(); + backOffPolicy.setInitialInterval(METADATA_VERIFICATION_RETRY_INITIAL_INTERVAL); + backOffPolicy.setMultiplier(METADATA_VERIFICATION_RETRY_BACKOFF_MULTIPLIER); + backOffPolicy.setMaxInterval(METADATA_VERIFICATION_MAX_INTERVAL); + retryTemplate.setBackOffPolicy(backOffPolicy); + retryOperations = retryTemplate; + } + } + + /** + * Allowed chars are ASCII alphanumerics, '.', '_' and '-'. '_' is used as escaped char in the form '_xx' where xx + * is the hexadecimal value of the byte(s) needed to represent an illegal char in utf8. + */ + /*default*/ + public static String escapeTopicName(String original) { + StringBuilder result = new StringBuilder(original.length()); + try { + byte[] utf8 = original.getBytes("UTF-8"); + for (byte b : utf8) { + if ((b >= 'a') && (b <= 'z') || (b >= 'A') && (b <= 'Z') || (b >= '0') && (b <= '9') || (b == '.') + || (b == '-')) { + result.append((char) b); + } + else { + result.append(String.format("_%02X", b)); + } + } + } + catch (UnsupportedEncodingException e) { + throw new AssertionError(e); // Can't happen + } + return result.toString(); + } + + public void setDefaultReplicationFactor(int defaultReplicationFactor) { + this.defaultReplicationFactor = defaultReplicationFactor; + } + + public void setDefaultCompressionCodec(String defaultCompressionCodec) { + this.defaultCompressionCodec = defaultCompressionCodec; + } + + public void setDefaultRequiredAcks(int defaultRequiredAcks) { + this.defaultRequiredAcks = defaultRequiredAcks; + } + + /** + * Set the default auto commit enabled property; This is used to commit the offset either automatically or + * manually. + * @param defaultAutoCommitEnabled + */ + public void setDefaultAutoCommitEnabled(boolean defaultAutoCommitEnabled) { + this.defaultAutoCommitEnabled = defaultAutoCommitEnabled; + } + + public void setDefaultQueueSize(int defaultQueueSize) { + this.defaultQueueSize = defaultQueueSize; + } + + public void setDefaultFetchSize(int defaultFetchSize) { + this.defaultFetchSize = defaultFetchSize; + } + + public void setDefaultMinPartitionCount(int defaultMinPartitionCount) { + this.defaultMinPartitionCount = defaultMinPartitionCount; + } + + public void setDefaultMaxWait(int defaultMaxWait) { + this.defaultMaxWait = defaultMaxWait; + } + + public void setMode(Mode mode) { + this.mode = mode; + } + + @Override + public void bindConsumer(String name, final MessageChannel moduleInputChannel, Properties properties) { + // Point-to-point consumers reset at the earliest time, which allows them to catch up with all messages + createKafkaConsumer(name, moduleInputChannel, properties, POINT_TO_POINT_SEMANTICS_CONSUMER_GROUP, + OffsetRequest.EarliestTime()); + bindExistingProducerDirectlyIfPossible(name, moduleInputChannel); + } + + @Override + public void bindPubSubConsumer(String name, MessageChannel inputChannel, Properties properties) { + // Usage of a different consumer group each time achieves pub-sub + // PubSub consumers reset at the latest time, which allows them to receive only messages sent after + // they've been bound + String group = UUID.randomUUID().toString(); + createKafkaConsumer(name, inputChannel, properties, group, OffsetRequest.LatestTime()); + } + + @Override + public void bindProducer(final String name, MessageChannel moduleOutputChannel, Properties properties) { + + Assert.isInstanceOf(SubscribableChannel.class, moduleOutputChannel); + KafkaPropertiesAccessor producerPropertiesAccessor = new KafkaPropertiesAccessor(properties); + if (name.startsWith(P2P_NAMED_CHANNEL_TYPE_PREFIX)) { + validateProducerProperties(name, properties, SUPPORTED_NAMED_PRODUCER_PROPERTIES); + } + else { + validateProducerProperties(name, properties, SUPPORTED_PRODUCER_PROPERTIES); + } + if (!bindNewProducerDirectlyIfPossible(name, (SubscribableChannel) moduleOutputChannel, + producerPropertiesAccessor)) { + if (logger.isInfoEnabled()) { + logger.info("Using kafka topic for outbound: " + name); + } + + final String topicName = escapeTopicName(name); + + int numPartitions = producerPropertiesAccessor.getNumberOfKafkaPartitionsForProducer(); + + Collection partitions = ensureTopicCreated(topicName, numPartitions, defaultReplicationFactor); + + ProducerMetadata producerMetadata = new ProducerMetadata<>( + topicName, byte[].class, byte[].class, BYTE_ARRAY_SERIALIZER, BYTE_ARRAY_SERIALIZER); + producerMetadata.setCompressionType(ProducerMetadata.CompressionType.valueOf( + producerPropertiesAccessor.getCompressionCodec(this.defaultCompressionCodec))); + producerMetadata.setBatchBytes(producerPropertiesAccessor.getBatchSize(this.defaultBatchSize)); + Properties additionalProps = new Properties(); + additionalProps.put(ProducerConfig.ACKS_CONFIG, + String.valueOf(producerPropertiesAccessor.getRequiredAcks(this + .defaultRequiredAcks))); + additionalProps.put(ProducerConfig.LINGER_MS_CONFIG, + String.valueOf(producerPropertiesAccessor.getBatchTimeout(this + .defaultBatchTimeout))); + ProducerFactoryBean producerFB = + new ProducerFactoryBean<>(producerMetadata, brokers, additionalProps); + + try { + final ProducerConfiguration producerConfiguration = new ProducerConfiguration<>( + producerMetadata, producerFB.getObject()); + + MessageHandler handler = new SendingHandler(topicName, producerPropertiesAccessor, + partitions.size(), producerConfiguration); + EventDrivenConsumer consumer = new EventDrivenConsumer((SubscribableChannel) moduleOutputChannel, + handler); + consumer.setBeanFactory(this.getBeanFactory()); + consumer.setBeanName("outbound." + name); + consumer.afterPropertiesSet(); + Binding producerBinding = Binding.forProducer(name, moduleOutputChannel, consumer, + producerPropertiesAccessor); + addBinding(producerBinding); + producerBinding.start(); + } + catch (Exception e) { + throw new RuntimeException(e); + } + + } + + } + + @Override + public void bindPubSubProducer(String name, MessageChannel outputChannel, Properties properties) { + bindProducer(name, outputChannel, properties); + } + + @Override + public void bindRequestor(String name, MessageChannel requests, MessageChannel replies, Properties properties) { + throw new UnsupportedOperationException("requestor binding is not supported by this bus"); + } + + @Override + public void bindReplier(String name, MessageChannel requests, MessageChannel replies, Properties properties) { + throw new UnsupportedOperationException("replier binding is not supported by this bus"); + } + + /** + * Creates a Kafka topic if needed, or try to increase its partition count to the desired number. + */ + private Collection ensureTopicCreated(final String topicName, final int numPartitions, + int replicationFactor) { + + final int sessionTimeoutMs = 10000; + final int connectionTimeoutMs = 10000; + final ZkClient zkClient = new ZkClient(zkAddress, sessionTimeoutMs, connectionTimeoutMs, utf8Serializer); + try { + // The following is basically copy/paste from AdminUtils.createTopic() with + // createOrUpdateTopicPartitionAssignmentPathInZK(..., update=true) + final Properties topicConfig = new Properties(); + Seq brokerList = ZkUtils.getSortedBrokerList(zkClient); + final scala.collection.Map> replicaAssignment = AdminUtils.assignReplicasToBrokers + (brokerList, + numPartitions, replicationFactor, -1, -1); + retryOperations.execute(new RetryCallback() { + + @Override + public Object doWithRetry(RetryContext context) throws RuntimeException { + AdminUtils.createOrUpdateTopicPartitionAssignmentPathInZK(zkClient, topicName, replicaAssignment, + topicConfig, true); + return null; + } + }); + try { + Collection partitions = retryOperations.execute(new RetryCallback, Exception>() { + + @Override + public Collection doWithRetry(RetryContext context) throws Exception { + connectionFactory.refreshMetadata(Collections.singleton(topicName)); + Collection partitions = connectionFactory.getPartitions(topicName); + if (partitions.size() < numPartitions) { + throw new IllegalStateException("The number of expected partitions was: " + numPartitions + + ", but " + + partitions.size() + " have been found instead"); + } + connectionFactory.getLeaders(partitions); + return partitions; + } + }); + return partitions; + } + catch (Exception e) { + logger.error("Cannot initialize MessageBus", e); + throw new RuntimeException("Cannot initialize message bus:", e); + } + + } + finally { + zkClient.close(); + } + } + + private void createKafkaConsumer(String name, final MessageChannel moduleInputChannel, Properties properties, + String group, long referencePoint) { + + if (name.startsWith(P2P_NAMED_CHANNEL_TYPE_PREFIX)) { + validateConsumerProperties(name, properties, SUPPORTED_NAMED_CONSUMER_PROPERTIES); + } + else { + validateConsumerProperties(name, properties, SUPPORTED_CONSUMER_PROPERTIES); + } + KafkaPropertiesAccessor accessor = new KafkaPropertiesAccessor(properties); + + int maxConcurrency = accessor.getConcurrency(defaultConcurrency); + + String topic = escapeTopicName(name); + + int numPartitions = accessor.getNumberOfKafkaPartitionsForConsumer(); + Collection allPartitions = ensureTopicCreated(topic, numPartitions, defaultReplicationFactor); + + Decoder valueDecoder = new DefaultDecoder(null); + Decoder keyDecoder = new DefaultDecoder(null); + + Collection listenedPartitions; + + int moduleCount = accessor.getCount(); + + if (moduleCount == 1) { + listenedPartitions = allPartitions; + } + else { + listenedPartitions = new ArrayList(); + for (Partition partition : allPartitions) { + // divide partitions across modules + if (accessor.getPartitionIndex() != -1) { + if ((partition.getId() % moduleCount) == accessor.getPartitionIndex()) { + listenedPartitions.add(partition); + } + } + else { + int moduleSequence = accessor.getSequence(); + if (moduleCount == 0) { + throw new IllegalArgumentException("The Kafka transport does not support 0-count modules"); + } + else { + // sequence numbers are zero-based + if ((partition.getId() % moduleCount) == (moduleSequence - 1)) { + listenedPartitions.add(partition); + } + } + } + } + } + + ReceivingHandler rh = new ReceivingHandler(); + rh.setOutputChannel(moduleInputChannel); + + final FixedSubscriberChannel bridge = new FixedSubscriberChannel(rh); + bridge.setBeanName("bridge." + name); + + final KafkaMessageListenerContainer messageListenerContainer = + createMessageListenerContainer(accessor, group, maxConcurrency, listenedPartitions, + referencePoint); + + final KafkaMessageDrivenChannelAdapter kafkaMessageDrivenChannelAdapter = + new KafkaMessageDrivenChannelAdapter(messageListenerContainer); + kafkaMessageDrivenChannelAdapter.setBeanFactory(this.getBeanFactory()); + kafkaMessageDrivenChannelAdapter.setKeyDecoder(keyDecoder); + kafkaMessageDrivenChannelAdapter.setPayloadDecoder(valueDecoder); + kafkaMessageDrivenChannelAdapter.setOutputChannel(bridge); + kafkaMessageDrivenChannelAdapter.setAutoCommitOffset(accessor.getDefaultAutoCommitEnabled(this + .defaultAutoCommitEnabled)); + kafkaMessageDrivenChannelAdapter.afterPropertiesSet(); + kafkaMessageDrivenChannelAdapter.start(); + + + EventDrivenConsumer edc = new EventDrivenConsumer(bridge, rh) { + + @Override + protected void doStop() { + // stop the offset manager and the channel adapter before unbinding + // this means that the upstream channel adapter has a chance to stop + kafkaMessageDrivenChannelAdapter.stop(); + if (messageListenerContainer.getOffsetManager() instanceof DisposableBean) { + try { + ((DisposableBean) messageListenerContainer.getOffsetManager()).destroy(); + } + catch (Exception e) { + logger.error("Error while closing the offset manager", e); + } + } + super.doStop(); + } + }; + edc.setBeanName("inbound." + name); + + Binding consumerBinding = Binding.forConsumer(name, edc, moduleInputChannel, accessor); + addBinding(consumerBinding); + consumerBinding.start(); + + } + + public KafkaMessageListenerContainer createMessageListenerContainer(Properties properties, String group, + int maxConcurrency, String topic, long referencePoint) { + return createMessageListenerContainer(new KafkaPropertiesAccessor(properties), group, maxConcurrency, topic, + null, referencePoint); + } + + private KafkaMessageListenerContainer createMessageListenerContainer(KafkaPropertiesAccessor accessor, + String group, int maxConcurrency, Collection listenedPartitions, long referencePoint) { + return createMessageListenerContainer(accessor, group, maxConcurrency, null, listenedPartitions, referencePoint); + } + + private KafkaMessageListenerContainer createMessageListenerContainer(KafkaPropertiesAccessor accessor, + String group, int maxConcurrency, String topic, Collection listenedPartitions, + long referencePoint) { + Assert.isTrue(StringUtils.hasText(topic) ^ !CollectionUtils.isEmpty(listenedPartitions), + "Exactly one of topic or a list of listened partitions must be provided"); + KafkaMessageListenerContainer messageListenerContainer; + if (topic != null) { + messageListenerContainer = new KafkaMessageListenerContainer(connectionFactory, topic); + } + else { + messageListenerContainer = new KafkaMessageListenerContainer(connectionFactory, + listenedPartitions.toArray(new Partition[listenedPartitions.size()])); + } + if (logger.isDebugEnabled()) { + logger.debug("Listening to topic " + topic); + } + // if we have less target partitions than target concurrency, adjust accordingly + messageListenerContainer.setConcurrency(Math.min(maxConcurrency, listenedPartitions.size())); + OffsetManager offsetManager = createOffsetManager(group, referencePoint); + messageListenerContainer.setOffsetManager(offsetManager); + messageListenerContainer.setQueueSize(accessor.getProperty(QUEUE_SIZE, defaultQueueSize)); + messageListenerContainer.setMaxFetch(accessor.getProperty(FETCH_SIZE, defaultFetchSize)); + return messageListenerContainer; + } + + private OffsetManager createOffsetManager(String group, long referencePoint) { + try { + KafkaTopicOffsetManager kafkaOffsetManager = + new KafkaTopicOffsetManager(zookeeperConnect, offsetStoreTopic, Collections. emptyMap()); + kafkaOffsetManager.setConsumerId(group); + kafkaOffsetManager.setReferenceTimestamp(referencePoint); + kafkaOffsetManager.setSegmentSize(offsetStoreSegmentSize); + kafkaOffsetManager.setRetentionTime(offsetStoreRetentionTime); + kafkaOffsetManager.setRequiredAcks(offsetStoreRequiredAcks); + kafkaOffsetManager.setMaxSize(offsetStoreMaxFetchSize); + kafkaOffsetManager.setBatchBytes(offsetStoreBatchBytes); + kafkaOffsetManager.setMaxQueueBufferingTime(offsetStoreBatchTime); + + kafkaOffsetManager.afterPropertiesSet(); + + WindowingOffsetManager windowingOffsetManager = new WindowingOffsetManager(kafkaOffsetManager); + windowingOffsetManager.setTimespan(offsetUpdateTimeWindow); + windowingOffsetManager.setCount(offsetUpdateCount); + windowingOffsetManager.setShutdownTimeout(offsetUpdateShutdownTimeout); + + windowingOffsetManager.afterPropertiesSet(); + return windowingOffsetManager; + } + catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public void doManualAck(LinkedList messageHeadersList) { + Iterator iterator = messageHeadersList.iterator(); + while (iterator.hasNext()) { + MessageHeaders headers = iterator.next(); + Acknowledgment acknowledgment = (Acknowledgment) headers.get(KafkaHeaders.ACKNOWLEDGMENT); + Assert.notNull(acknowledgment, "Acknowledgement shouldn't be null when acknowledging kafka message " + + "manually."); + acknowledgment.acknowledge(); + } + } + + private class KafkaPropertiesAccessor extends AbstractBusPropertiesAccessor { + + public KafkaPropertiesAccessor(Properties properties) { + super(properties); + } + + public int getNumberOfKafkaPartitionsForProducer() { + int nextModuleCount = getNextModuleCount(); + if (nextModuleCount == 0) { + throw new IllegalArgumentException("Module count cannot be zero"); + } + int nextModuleConcurrency = getProperty(NEXT_MODULE_CONCURRENCY, defaultConcurrency); + int minKafkaPartitions = getMinPartitionCount(defaultMinPartitionCount); + return Math.max(minKafkaPartitions, nextModuleCount * nextModuleConcurrency); + } + + public int getNumberOfKafkaPartitionsForConsumer() { + int concurrency = getConcurrency(defaultConcurrency); + int minKafkaPartitions = getMinPartitionCount(defaultMinPartitionCount); + int moduleCount = getCount(); + if (moduleCount == 0) { + throw new IllegalArgumentException("Module count cannot be zero"); + } + return Math.max(minKafkaPartitions, moduleCount * concurrency); + } + + public String getCompressionCodec(String defaultValue) { + return getProperty(COMPRESSION_CODEC, defaultValue); + } + + public int getRequiredAcks(int defaultRequiredAcks) { + return getProperty(REQUIRED_ACKS, defaultRequiredAcks); + } + + public boolean getDefaultAutoCommitEnabled(boolean defaultAutoCommitEnabled) { + return getProperty(AUTO_COMMIT_ENABLED, defaultAutoCommitEnabled); + } + + public int getMinPartitionCount(int defaultPartitionCount) { + return getProperty(MIN_PARTITION_COUNT, defaultPartitionCount); + } + + } + + private class ReceivingHandler extends AbstractReplyProducingMessageHandler { + + public ReceivingHandler() { + this.setBeanFactory(KafkaMessageBus.this.getBeanFactory()); + } + + @Override + @SuppressWarnings("unchecked") + protected Object handleRequestMessage(Message requestMessage) { + if (Mode.embeddedHeaders.equals(mode)) { + MessageValues messageValues; + try { + messageValues = embeddedHeadersMessageConverter.extractHeaders((Message) requestMessage, + true); + } + catch (Exception e) { + logger.error(EmbeddedHeadersMessageConverter.decodeExceptionMessage(requestMessage), e); + messageValues = new MessageValues(requestMessage); + } + messageValues = deserializePayloadIfNecessary(messageValues); + return MessageBuilder.createMessage(messageValues.getPayload(), new KafkaBusMessageHeaders( + messageValues)); + } + else { + return requestMessage; + } + } + + @SuppressWarnings("serial") + private final class KafkaBusMessageHeaders extends MessageHeaders { + + KafkaBusMessageHeaders(Map headers) { + super(headers, MessageHeaders.ID_VALUE_NONE, -1L); + } + } + + + @Override + protected boolean shouldCopyRequestHeaders() { + // prevent the message from being copied again in superclass + return false; + } + } + + private class SendingHandler extends AbstractMessageHandler { + + private final PartitioningMetadata partitioningMetadata; + + private final AtomicInteger roundRobinCount = new AtomicInteger(); + + private final String topicName; + + private final int numberOfKafkaPartitions; + + private final ProducerConfiguration producerConfiguration; + + + private SendingHandler(String topicName, KafkaPropertiesAccessor properties, int numberOfPartitions, + ProducerConfiguration producerConfiguration) { + this.topicName = topicName; + this.numberOfKafkaPartitions = numberOfPartitions; + this.partitioningMetadata = new PartitioningMetadata(properties, numberOfPartitions); + this.setBeanFactory(KafkaMessageBus.this.getBeanFactory()); + this.producerConfiguration = producerConfiguration; + } + + @Override + protected void handleMessageInternal(Message message) throws Exception { + int targetPartition; + if (partitioningMetadata.isPartitionedModule()) { + targetPartition = determinePartition(message, partitioningMetadata); + } + else { + targetPartition = roundRobin() % numberOfKafkaPartitions; + } + + if (Mode.embeddedHeaders.equals(mode)) { + MessageValues transformed = serializePayloadIfNecessary(message); + byte[] messageToSend = embeddedHeadersMessageConverter.embedHeaders(transformed, + KafkaMessageBus.this.headersToMap); + producerConfiguration.send(topicName, targetPartition, null, messageToSend); + } + else if (Mode.raw.equals(mode)) { + Object contentType = message.getHeaders().get(MessageHeaders.CONTENT_TYPE); + if (contentType != null + && !contentType.equals(MediaType.APPLICATION_OCTET_STREAM_VALUE)) { + logger.error("Raw mode supports only " + MediaType.APPLICATION_OCTET_STREAM_VALUE + " content type" + + message.getPayload().getClass()); + } + if (message.getPayload() instanceof byte[]) { + producerConfiguration.send(topicName, targetPartition, null, (byte[]) message.getPayload()); + } + else { + logger.error("Raw mode supports only byte[] payloads but value sent was of type " + + message.getPayload().getClass()); + } + } + } + + private int roundRobin() { + int result = roundRobinCount.incrementAndGet(); + if (result == Integer.MAX_VALUE) { + roundRobinCount.set(0); + } + return result; + } + + } + + public enum Mode { + raw, + embeddedHeaders + } + +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-kafka/src/main/java/org/springframework/xd/dirt/integration/kafka/WindowingOffsetManager.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-kafka/src/main/java/org/springframework/xd/dirt/integration/kafka/WindowingOffsetManager.java new file mode 100644 index 000000000..6d2af4751 --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-kafka/src/main/java/org/springframework/xd/dirt/integration/kafka/WindowingOffsetManager.java @@ -0,0 +1,263 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.xd.dirt.integration.kafka; + +import java.io.IOException; +import java.util.Collection; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.integration.kafka.core.Partition; +import org.springframework.integration.kafka.listener.OffsetManager; +import org.springframework.util.Assert; + +import rx.Observable; +import rx.Subscription; +import rx.functions.Action0; +import rx.functions.Action1; +import rx.functions.Func1; +import rx.functions.Func2; +import rx.observables.GroupedObservable; +import rx.observables.MathObservable; +import rx.subjects.PublishSubject; +import rx.subjects.SerializedSubject; +import rx.subjects.Subject; + +/** + * An {@link OffsetManager} that aggregates writes over a time or count window, using an underlying delegate to + * do the actual operations. Its purpose is to reduce the performance impact of writing operations + * wherever this is desirable. + * + * Either a time window or a number of writes can be specified, but not both. + * + * @author Marius Bogoevici + */ +//TODO: Move this class to spring-integration-kafka +public class WindowingOffsetManager implements OffsetManager, InitializingBean, DisposableBean { + + private final CreatePartitionAndOffsetFunction createPartitionAndOffsetFunction = new CreatePartitionAndOffsetFunction(); + + private final GetOffsetFunction getOffsetFunction = new GetOffsetFunction(); + + private final ComputeMaximumOffsetByPartitionFunction findHighestOffsetInPartitionGroup = new ComputeMaximumOffsetByPartitionFunction(); + + private final GetPartitionFunction getPartition = new GetPartitionFunction(); + + private final FindHighestOffsetsByPartitionFunction findHighestOffsetsByPartition = new FindHighestOffsetsByPartitionFunction(); + + private final DelegateUpdateOffsetAction delegateUpdateOffsetAction = new DelegateUpdateOffsetAction(); + + private final NotifyObservableClosedAction notifyObservableClosed = new NotifyObservableClosedAction(); + + private final OffsetManager delegate; + + private long timespan = 10 * 1000; + + private int count = 0; + + private Subject offsets; + + private Subscription subscription; + + private int shutdownTimeout = 2000; + + private CountDownLatch shutdownLatch; + + public WindowingOffsetManager(OffsetManager offsetManager) { + this.delegate = offsetManager; + } + + /** + * The timespan for aggregating write operations, before invoking the underlying {@link OffsetManager}. + * + * @param timespan duration in milliseconds + */ + public void setTimespan(long timespan) { + Assert.isTrue(timespan >= 0, "Timespan must be a positive value"); + this.timespan = timespan; + } + + /** + * How many writes should be aggregated, before invoking the underlying {@link OffsetManager}. Setting this value + * to 1 effectively disables windowing. + * + * @param count number of writes + */ + public void setCount(int count) { + Assert.isTrue(count >= 0, "Count must be a positive value"); + this.count = count; + } + + /** + * The timeout that {@link #close()} and {@link #destroy()} operations will wait for receving a confirmation that the + * underlying writes have been processed. + * + * @param shutdownTimeout duration in milliseconds + */ + public void setShutdownTimeout(int shutdownTimeout) { + this.shutdownTimeout = shutdownTimeout; + } + + @Override + public void afterPropertiesSet() throws Exception { + Assert.isTrue(timespan > 0 ^ count > 0, "Only one of the timespan or count must be set"); + // create the stream if windowing is set, and count is higher than 1 + if (timespan > 0 || count > 1) { + offsets = new SerializedSubject<>(PublishSubject.create()); + // window by either count or time + Observable> window = + timespan > 0 ? offsets.window(timespan, TimeUnit.MILLISECONDS) : offsets.window(count); + Observable maximumOffsetsByWindow = window + .flatMap(findHighestOffsetsByPartition) + .doOnCompleted(notifyObservableClosed); + subscription = maximumOffsetsByWindow.subscribe(delegateUpdateOffsetAction); + } + else { + offsets = null; + } + } + + @Override + public void destroy() throws Exception { + this.flush(); + this.close(); + if (delegate instanceof DisposableBean) { + ((DisposableBean) delegate).destroy(); + } + } + + @Override + public void updateOffset(Partition partition, long offset) { + if (offsets != null) { + offsets.onNext(new PartitionAndOffset(partition, offset)); + } + else { + delegate.updateOffset(partition, offset); + } + } + + @Override + public long getOffset(Partition partition) { + return delegate.getOffset(partition); + } + + @Override + public void deleteOffset(Partition partition) { + delegate.deleteOffset(partition); + } + + @Override + public void resetOffsets(Collection partition) { + delegate.resetOffsets(partition); + } + + @Override + public void close() throws IOException { + if (offsets != null) { + shutdownLatch = new CountDownLatch(1); + offsets.onCompleted(); + try { + shutdownLatch.await(shutdownTimeout, TimeUnit.MILLISECONDS); + } + catch (InterruptedException e) { + // ignore + } + subscription.unsubscribe(); + } + delegate.close(); + } + + @Override + public void flush() throws IOException { + delegate.flush(); + } + + class PartitionAndOffset { + + private final Partition partition; + + private final Long offset; + + public PartitionAndOffset(Partition partition, Long offset) { + this.partition = partition; + this.offset = offset; + } + + public Partition getPartition() { + return partition; + } + + public Long getOffset() { + return offset; + } + } + + private class DelegateUpdateOffsetAction implements Action1 { + @Override + public void call(PartitionAndOffset partitionAndOffset) { + delegate.updateOffset(partitionAndOffset.getPartition(), partitionAndOffset.getOffset()); + } + } + + private class NotifyObservableClosedAction implements Action0 { + @Override + public void call() { + if (shutdownLatch != null) { + shutdownLatch.countDown(); + } + } + } + + private class CreatePartitionAndOffsetFunction implements Func2 { + @Override + public PartitionAndOffset call(Partition partition, Long offset) { + return new PartitionAndOffset(partition, offset); + } + } + + private class GetOffsetFunction implements Func1 { + @Override + public Long call(PartitionAndOffset partitionAndOffset) { + return partitionAndOffset.getOffset(); + } + } + + private class ComputeMaximumOffsetByPartitionFunction implements Func1, Observable> { + @Override + public Observable call(GroupedObservable group) { + return Observable.zip(Observable.just(group.getKey()), + MathObservable.max(group.map(getOffsetFunction)), + createPartitionAndOffsetFunction); + } + } + + private class GetPartitionFunction implements Func1 { + @Override + public Partition call(PartitionAndOffset partitionAndOffset) { + return partitionAndOffset.getPartition(); + } + } + + private class FindHighestOffsetsByPartitionFunction implements Func1, Observable> { + @Override + public Observable call(Observable windowBuffer) { + return windowBuffer.groupBy(getPartition).flatMap(findHighestOffsetInPartitionGroup); + } + } +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-kafka/src/main/resources/META-INF/spring-xd/bus/kafka-bus.xml b/spring-cloud-streams-bindings/spring-cloud-streams-binding-kafka/src/main/resources/META-INF/spring-xd/bus/kafka-bus.xml new file mode 100644 index 000000000..eabf74718 --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-kafka/src/main/resources/META-INF/spring-xd/bus/kafka-bus.xml @@ -0,0 +1,49 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-kafka/src/test/java/org/springframework/xd/dirt/integration/bus/kafka/KafkaMessageBusTests.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-kafka/src/test/java/org/springframework/xd/dirt/integration/bus/kafka/KafkaMessageBusTests.java new file mode 100644 index 000000000..0e9a5c270 --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-kafka/src/test/java/org/springframework/xd/dirt/integration/bus/kafka/KafkaMessageBusTests.java @@ -0,0 +1,315 @@ +/* + * Copyright 2014-2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.xd.dirt.integration.bus.kafka; + +import static org.hamcrest.collection.IsCollectionWithSize.hasSize; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; + +import java.util.Arrays; +import java.util.Collection; +import java.util.Properties; +import java.util.UUID; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.TimeUnit; + +import kafka.api.OffsetRequest; + +import org.junit.Ignore; +import org.junit.Rule; +import org.junit.Test; + +import org.springframework.integration.channel.DirectChannel; +import org.springframework.integration.channel.QueueChannel; +import org.springframework.integration.kafka.core.KafkaMessage; +import org.springframework.integration.kafka.core.Partition; +import org.springframework.integration.kafka.listener.KafkaMessageListenerContainer; +import org.springframework.integration.kafka.listener.MessageListener; +import org.springframework.messaging.Message; +import org.springframework.xd.dirt.integration.bus.BusProperties; +import org.springframework.xd.dirt.integration.bus.MessageBus; +import org.springframework.xd.dirt.integration.bus.PartitionCapableBusTests; +import org.springframework.xd.dirt.integration.bus.Spy; +import org.springframework.xd.dirt.integration.kafka.KafkaMessageBus; +import org.springframework.xd.dirt.integration.kafka.KafkaTestSupport; + + +/** + * Integration tests for the {@link KafkaMessageBus}. + * + * @author Eric Bottard + * @author Marius Bogoevici + */ +@Ignore //TODO: Fix this test +public class KafkaMessageBusTests extends PartitionCapableBusTests { + + @Rule + public KafkaTestSupport kafkaTestSupport = new KafkaTestSupport(); + + private KafkaTestMessageBus messageBus; + + @Override + protected void busBindUnbindLatency() throws InterruptedException { + Thread.sleep(500); + } + + @Override + protected MessageBus getMessageBus() { + if (messageBus == null) { + messageBus = createKafkaTestMessageBus(); + } + return messageBus; + } + + protected KafkaTestMessageBus createKafkaTestMessageBus() { + return new KafkaTestMessageBus(kafkaTestSupport, getCodec(), KafkaMessageBus.Mode.embeddedHeaders); + } + + @Override + protected boolean usesExplicitRouting() { + return false; + } + + @Override + public Spy spyOn(final String name) { + String topic = KafkaMessageBus.escapeTopicName(name); + + KafkaTestMessageBus busWrapper = (KafkaTestMessageBus) getMessageBus(); + // Rewind offset, as tests will have typically already sent the messages we're trying to consume + + KafkaMessageListenerContainer messageListenerContainer = busWrapper.getCoreMessageBus().createMessageListenerContainer( + new Properties(), UUID.randomUUID().toString(), 1, topic, OffsetRequest.EarliestTime()); + + final BlockingQueue messages = new ArrayBlockingQueue(10); + + messageListenerContainer.setMessageListener(new MessageListener() { + + @Override + public void onMessage(KafkaMessage message) { + messages.offer(message); + } + }); + + + return new Spy() { + + @Override + public Object receive(boolean expectNull) throws Exception { + return messages.poll(expectNull ? 50 : 5000, TimeUnit.MILLISECONDS); + } + }; + + } + + @Test + public void testCompression() throws Exception { + final String[] codecs = new String[] { null, "none", "gzip", "snappy" }; + + byte[] ratherBigPayload = new byte[2048]; + Arrays.fill(ratherBigPayload, (byte) 65); + MessageBus messageBus = getMessageBus(); + + for (String codec : codecs) { + DirectChannel moduleOutputChannel = new DirectChannel(); + QueueChannel moduleInputChannel = new QueueChannel(); + Properties props = new Properties(); + if (codec != null) { + props.put(KafkaMessageBus.COMPRESSION_CODEC, codec); + } + messageBus.bindProducer("foo.0", moduleOutputChannel, props); + messageBus.bindConsumer("foo.0", moduleInputChannel, null); + Message message = org.springframework.integration.support.MessageBuilder.withPayload(ratherBigPayload).build(); + // Let the consumer actually bind to the producer before sending a msg + busBindUnbindLatency(); + moduleOutputChannel.send(message); + Message inbound = moduleInputChannel.receive(2000); + assertNotNull(inbound); + assertArrayEquals(ratherBigPayload, (byte[]) inbound.getPayload()); + messageBus.unbindProducers("foo.0"); + messageBus.unbindConsumers("foo.0"); + } + } + + @Test + public void testCustomPartitionCountOverridesDefaultIfLarger() throws Exception { + + byte[] ratherBigPayload = new byte[2048]; + Arrays.fill(ratherBigPayload, (byte) 65); + KafkaTestMessageBus messageBus = (KafkaTestMessageBus) getMessageBus(); + + + DirectChannel moduleOutputChannel = new DirectChannel(); + QueueChannel moduleInputChannel = new QueueChannel(); + Properties producerProperties = new Properties(); + producerProperties.put(BusProperties.MIN_PARTITION_COUNT, "10"); + Properties consumerProperties = new Properties(); + consumerProperties.put(BusProperties.MIN_PARTITION_COUNT, "10"); + long uniqueBindingId = System.currentTimeMillis(); + messageBus.bindProducer("foo" + uniqueBindingId + ".0", moduleOutputChannel, producerProperties); + messageBus.bindConsumer("foo" + uniqueBindingId + ".0", moduleInputChannel, consumerProperties); + Message message = org.springframework.integration.support.MessageBuilder.withPayload(ratherBigPayload).build(); + // Let the consumer actually bind to the producer before sending a msg + busBindUnbindLatency(); + moduleOutputChannel.send(message); + Message inbound = moduleInputChannel.receive(2000); + assertNotNull(inbound); + assertArrayEquals(ratherBigPayload, (byte[]) inbound.getPayload()); + Collection partitions = messageBus.getCoreMessageBus().getConnectionFactory().getPartitions( + "foo" + uniqueBindingId + ".0"); + assertThat(partitions, hasSize(10)); + messageBus.unbindProducers("foo" + uniqueBindingId + ".0"); + messageBus.unbindConsumers("foo" + uniqueBindingId + ".0"); + } + + @Test + public void testCustomPartitionCountDoesNotOverrideModuleCountAndConcurrencyIfSmaller() throws Exception { + + byte[] ratherBigPayload = new byte[2048]; + Arrays.fill(ratherBigPayload, (byte) 65); + KafkaTestMessageBus messageBus = (KafkaTestMessageBus) getMessageBus(); + + + DirectChannel moduleOutputChannel = new DirectChannel(); + QueueChannel moduleInputChannel = new QueueChannel(); + Properties producerProps = new Properties(); + producerProps.put(BusProperties.MIN_PARTITION_COUNT, "5"); + producerProps.put(BusProperties.NEXT_MODULE_CONCURRENCY, "6"); + Properties consumerProps = new Properties(); + consumerProps.put(BusProperties.MIN_PARTITION_COUNT, "5"); + consumerProps.put(BusProperties.CONCURRENCY, "6"); + long uniqueBindingId = System.currentTimeMillis(); + messageBus.bindProducer("foo" + uniqueBindingId + ".0", moduleOutputChannel, producerProps); + messageBus.bindConsumer("foo" + uniqueBindingId + ".0", moduleInputChannel, consumerProps); + Message message = org.springframework.integration.support.MessageBuilder.withPayload(ratherBigPayload).build(); + // Let the consumer actually bind to the producer before sending a msg + busBindUnbindLatency(); + moduleOutputChannel.send(message); + Message inbound = moduleInputChannel.receive(2000); + assertNotNull(inbound); + assertArrayEquals(ratherBigPayload, (byte[]) inbound.getPayload()); + Collection partitions = messageBus.getCoreMessageBus().getConnectionFactory().getPartitions( + "foo" + uniqueBindingId + ".0"); + assertThat(partitions, hasSize(6)); + messageBus.unbindProducers("foo" + uniqueBindingId + ".0"); + messageBus.unbindConsumers("foo" + uniqueBindingId + ".0"); + } + + @Test + public void testCustomPartitionCountOverridesModuleCountAndConcurrencyIfLarger() throws Exception { + + byte[] ratherBigPayload = new byte[2048]; + Arrays.fill(ratherBigPayload, (byte) 65); + KafkaTestMessageBus messageBus = (KafkaTestMessageBus) getMessageBus(); + + DirectChannel moduleOutputChannel = new DirectChannel(); + QueueChannel moduleInputChannel = new QueueChannel(); + Properties producerProps = new Properties(); + producerProps.put(BusProperties.MIN_PARTITION_COUNT, "6"); + producerProps.put(BusProperties.NEXT_MODULE_CONCURRENCY, "5"); + Properties consumerProps = new Properties(); + consumerProps.put(BusProperties.MIN_PARTITION_COUNT, "6"); + consumerProps.put(BusProperties.CONCURRENCY, "5"); + long uniqueBindingId = System.currentTimeMillis(); + messageBus.bindProducer("foo" + uniqueBindingId + ".0", moduleOutputChannel, producerProps); + messageBus.bindConsumer("foo" + uniqueBindingId + ".0", moduleInputChannel, consumerProps); + Message message = org.springframework.integration.support.MessageBuilder.withPayload(ratherBigPayload).build(); + // Let the consumer actually bind to the producer before sending a msg + busBindUnbindLatency(); + moduleOutputChannel.send(message); + Message inbound = moduleInputChannel.receive(2000); + assertNotNull(inbound); + assertArrayEquals(ratherBigPayload, (byte[]) inbound.getPayload()); + Collection partitions = messageBus.getCoreMessageBus().getConnectionFactory().getPartitions( + "foo" + uniqueBindingId + ".0"); + assertThat(partitions, hasSize(6)); + messageBus.unbindProducers("foo" + uniqueBindingId + ".0"); + messageBus.unbindConsumers("foo" + uniqueBindingId + ".0"); + } + + @Test + public void testCustomPartitionCountDoesNotOverridePartitioningIfSmaller() throws Exception { + + byte[] ratherBigPayload = new byte[2048]; + Arrays.fill(ratherBigPayload, (byte) 65); + KafkaTestMessageBus messageBus = (KafkaTestMessageBus) getMessageBus(); + + DirectChannel moduleOutputChannel = new DirectChannel(); + QueueChannel moduleInputChannel = new QueueChannel(); + Properties producerProperties = new Properties(); + producerProperties.put(BusProperties.MIN_PARTITION_COUNT, "3"); + producerProperties.put(BusProperties.NEXT_MODULE_COUNT, "5"); + producerProperties.put(BusProperties.PARTITION_KEY_EXPRESSION, "payload"); + Properties consumerProperties = new Properties(); + consumerProperties.put(BusProperties.MIN_PARTITION_COUNT, "3"); + long uniqueBindingId = System.currentTimeMillis(); + messageBus.bindProducer("foo" + uniqueBindingId + ".0", moduleOutputChannel, producerProperties); + messageBus.bindConsumer("foo" + uniqueBindingId + ".0", moduleInputChannel, consumerProperties); + Message message = org.springframework.integration.support.MessageBuilder.withPayload(ratherBigPayload).build(); + // Let the consumer actually bind to the producer before sending a msg + busBindUnbindLatency(); + moduleOutputChannel.send(message); + Message inbound = moduleInputChannel.receive(2000); + assertNotNull(inbound); + assertArrayEquals(ratherBigPayload, (byte[]) inbound.getPayload()); + Collection partitions = messageBus.getCoreMessageBus().getConnectionFactory().getPartitions( + "foo" + uniqueBindingId + ".0"); + assertThat(partitions, hasSize(5)); + messageBus.unbindProducers("foo" + uniqueBindingId + ".0"); + messageBus.unbindConsumers("foo" + uniqueBindingId + ".0"); + } + + @Test + public void testCustomPartitionCountOverridesPartitioningIfLarger() throws Exception { + + byte[] ratherBigPayload = new byte[2048]; + Arrays.fill(ratherBigPayload, (byte) 65); + KafkaTestMessageBus messageBus = (KafkaTestMessageBus) getMessageBus(); + + DirectChannel moduleOutputChannel = new DirectChannel(); + QueueChannel moduleInputChannel = new QueueChannel(); + Properties producerProperties = new Properties(); + producerProperties.put(BusProperties.MIN_PARTITION_COUNT, "5"); + producerProperties.put(BusProperties.NEXT_MODULE_COUNT, "3"); + producerProperties.put(BusProperties.PARTITION_KEY_EXPRESSION, "payload"); + Properties consumerProperties = new Properties(); + consumerProperties.put(BusProperties.MIN_PARTITION_COUNT, "5"); + long uniqueBindingId = System.currentTimeMillis(); + messageBus.bindProducer("foo" + uniqueBindingId + ".0", moduleOutputChannel, producerProperties); + messageBus.bindConsumer("foo" + uniqueBindingId + ".0", moduleInputChannel, consumerProperties); + Message message = org.springframework.integration.support.MessageBuilder.withPayload(ratherBigPayload).build(); + // Let the consumer actually bind to the producer before sending a msg + busBindUnbindLatency(); + moduleOutputChannel.send(message); + Message inbound = moduleInputChannel.receive(2000); + assertNotNull(inbound); + assertArrayEquals(ratherBigPayload, (byte[]) inbound.getPayload()); + Collection partitions = messageBus.getCoreMessageBus().getConnectionFactory().getPartitions( + "foo" + uniqueBindingId + ".0"); + assertThat(partitions, hasSize(5)); + messageBus.unbindProducers("foo" + uniqueBindingId + ".0"); + messageBus.unbindConsumers("foo" + uniqueBindingId + ".0"); + } + + @Test + @Ignore("Kafka message bus does not support direct binding") + @Override + public void testDirectBinding() throws Exception { + + } +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-kafka/src/test/java/org/springframework/xd/dirt/integration/bus/kafka/KafkaTestMessageBus.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-kafka/src/test/java/org/springframework/xd/dirt/integration/bus/kafka/KafkaTestMessageBus.java new file mode 100644 index 000000000..115313645 --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-kafka/src/test/java/org/springframework/xd/dirt/integration/bus/kafka/KafkaTestMessageBus.java @@ -0,0 +1,76 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.xd.dirt.integration.bus.kafka; + +import org.springframework.context.support.GenericApplicationContext; +import org.springframework.integration.kafka.support.ZookeeperConnect; +import org.springframework.xd.dirt.integration.bus.AbstractTestMessageBus; +import org.springframework.xd.dirt.integration.bus.serializer.MultiTypeCodec; +import org.springframework.xd.dirt.integration.bus.serializer.kryo.PojoCodec; +import org.springframework.xd.dirt.integration.kafka.KafkaMessageBus; +import org.springframework.xd.dirt.integration.kafka.TestKafkaCluster; +import org.springframework.xd.dirt.integration.kafka.KafkaTestSupport; +import org.springframework.xd.tuple.serializer.kryo.TupleKryoRegistrar; + + +/** + * Test support class for {@link KafkaMessageBus}. + * Creates a bus that uses a test {@link TestKafkaCluster kafka cluster}. + * @author Eric Bottard + * @author Marius Bogoevici + * @author David Turanski + */ +public class KafkaTestMessageBus extends AbstractTestMessageBus { + + public KafkaTestMessageBus(KafkaTestSupport kafkaTestSupport) { + this(kafkaTestSupport, getCodec(), KafkaMessageBus.Mode.embeddedHeaders); + } + + + public KafkaTestMessageBus(KafkaTestSupport kafkaTestSupport, MultiTypeCodec codec, + KafkaMessageBus.Mode mode) { + + try { + ZookeeperConnect zookeeperConnect = new ZookeeperConnect(); + zookeeperConnect.setZkConnect(kafkaTestSupport.getZkConnectString()); + KafkaMessageBus messageBus = new KafkaMessageBus(zookeeperConnect, + kafkaTestSupport.getBrokerAddress(), + kafkaTestSupport.getZkConnectString(), codec); + messageBus.setDefaultBatchingEnabled(false); + messageBus.setMode(mode); + messageBus.afterPropertiesSet(); + GenericApplicationContext context = new GenericApplicationContext(); + context.refresh(); + messageBus.setApplicationContext(context); + this.setMessageBus(messageBus); + } + catch (Exception e) { + throw new RuntimeException(e); + } + } + + @Override + public void cleanup() { + // do nothing - the rule will take care of that + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + private static MultiTypeCodec getCodec() { + return new PojoCodec(new TupleKryoRegistrar()); + } + +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-kafka/src/test/java/org/springframework/xd/dirt/integration/bus/kafka/RawKafkaPartitionTestSupport.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-kafka/src/test/java/org/springframework/xd/dirt/integration/bus/kafka/RawKafkaPartitionTestSupport.java new file mode 100644 index 000000000..cf55c15f7 --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-kafka/src/test/java/org/springframework/xd/dirt/integration/bus/kafka/RawKafkaPartitionTestSupport.java @@ -0,0 +1,40 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.xd.dirt.integration.bus.kafka; + +import org.springframework.messaging.Message; +import org.springframework.xd.dirt.integration.bus.PartitionKeyExtractorStrategy; +import org.springframework.xd.dirt.integration.bus.PartitionSelectorStrategy; + + +/** + * + * @author Marius Bogoevici + */ +public class RawKafkaPartitionTestSupport implements PartitionKeyExtractorStrategy, PartitionSelectorStrategy { + + @Override + public int selectPartition(Object key, int divisor) { + return ((byte[])key)[0] % divisor; + } + + @Override + public Object extractKey(Message message) { + return message.getPayload(); + } + +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-kafka/src/test/java/org/springframework/xd/dirt/integration/bus/kafka/RawModeKafkaMessageBusTests.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-kafka/src/test/java/org/springframework/xd/dirt/integration/bus/kafka/RawModeKafkaMessageBusTests.java new file mode 100644 index 000000000..f0bf1f864 --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-kafka/src/test/java/org/springframework/xd/dirt/integration/bus/kafka/RawModeKafkaMessageBusTests.java @@ -0,0 +1,337 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.xd.dirt.integration.bus.kafka; + +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.containsString; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; + +import java.util.Arrays; +import java.util.List; +import java.util.Properties; + +import org.junit.Ignore; +import org.junit.Test; + +import org.springframework.integration.IntegrationMessageHeaderAccessor; +import org.springframework.integration.channel.DirectChannel; +import org.springframework.integration.channel.QueueChannel; +import org.springframework.integration.channel.interceptor.WireTap; +import org.springframework.integration.endpoint.AbstractEndpoint; +import org.springframework.integration.support.MessageBuilder; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageHeaders; +import org.springframework.messaging.support.GenericMessage; +import org.springframework.xd.dirt.integration.bus.Binding; +import org.springframework.xd.dirt.integration.bus.BusProperties; +import org.springframework.xd.dirt.integration.bus.MessageBus; +import org.springframework.xd.dirt.integration.bus.XdHeaders; +import org.springframework.xd.dirt.integration.kafka.KafkaMessageBus; +import org.springframework.xd.test.TestUtils; + +/** + * @author Marius Bogoevici + */ +@Ignore //TODO: Fix this test +public class RawModeKafkaMessageBusTests extends KafkaMessageBusTests { + + @Override + protected KafkaTestMessageBus createKafkaTestMessageBus() { + return new KafkaTestMessageBus(kafkaTestSupport, getCodec(), KafkaMessageBus.Mode.raw); + } + + @Test + @Override + public void testPartitionedModuleJava() throws Exception { + MessageBus bus = getMessageBus(); + Properties properties = new Properties(); + properties.put("partitionKeyExtractorClass", "org.springframework.xd.dirt.integration.bus.kafka.RawKafkaPartitionTestSupport"); + properties.put("partitionSelectorClass", "org.springframework.xd.dirt.integration.bus.kafka.RawKafkaPartitionTestSupport"); + properties.put(BusProperties.NEXT_MODULE_COUNT, "3"); + properties.put(BusProperties.NEXT_MODULE_CONCURRENCY, "2"); + + DirectChannel output = new DirectChannel(); + output.setBeanName("test.output"); + bus.bindProducer("partJ.0", output, properties); + @SuppressWarnings("unchecked") + List bindings = TestUtils.getPropertyValue(bus, "messageBus.bindings", List.class); + assertEquals(1, bindings.size()); + + properties.clear(); + properties.put("concurrency", "2"); + properties.put("count","3"); + properties.put("partitionIndex", "0"); + QueueChannel input0 = new QueueChannel(); + input0.setBeanName("test.input0J"); + bus.bindConsumer("partJ.0", input0, properties); + properties.put("partitionIndex", "1"); + QueueChannel input1 = new QueueChannel(); + input1.setBeanName("test.input1J"); + bus.bindConsumer("partJ.0", input1, properties); + properties.put("partitionIndex", "2"); + QueueChannel input2 = new QueueChannel(); + input2.setBeanName("test.input2J"); + bus.bindConsumer("partJ.0", input2, properties); + + output.send(new GenericMessage<>(new byte[]{(byte)0})); + output.send(new GenericMessage<>(new byte[]{(byte)1})); + output.send(new GenericMessage<>(new byte[]{(byte)2})); + + Message receive0 = input0.receive(1000); + assertNotNull(receive0); + Message receive1 = input1.receive(1000); + assertNotNull(receive1); + Message receive2 = input2.receive(1000); + assertNotNull(receive2); + + assertThat(Arrays.asList( + ((byte[]) receive0.getPayload())[0], + ((byte[]) receive1.getPayload())[0], + ((byte[]) receive2.getPayload())[0]), + containsInAnyOrder((byte)0, (byte)1, (byte)2)); + + bus.unbindConsumers("partJ.0"); + bus.unbindProducers("partJ.0"); + } + + @Test + @Override + public void testPartitionedModuleSpEL() throws Exception { + MessageBus bus = getMessageBus(); + Properties properties = new Properties(); + properties.put("partitionKeyExpression", "payload[0]"); + properties.put("partitionSelectorExpression", "hashCode()"); + properties.put(BusProperties.NEXT_MODULE_COUNT, "3"); + properties.put(BusProperties.NEXT_MODULE_CONCURRENCY, "2"); + + DirectChannel output = new DirectChannel(); + output.setBeanName("test.output"); + bus.bindProducer("part.0", output, properties); + @SuppressWarnings("unchecked") + List bindings = TestUtils.getPropertyValue(bus, "messageBus.bindings", List.class); + assertEquals(1, bindings.size()); + try { + AbstractEndpoint endpoint = bindings.get(0).getEndpoint(); + assertThat(getEndpointRouting(endpoint), containsString("part.0-' + headers['partition']")); + } + catch (UnsupportedOperationException ignored) { + + } + + properties.clear(); + properties.put("concurrency", "2"); + properties.put("partitionIndex", "0"); + properties.put("count","3"); + QueueChannel input0 = new QueueChannel(); + input0.setBeanName("test.input0S"); + bus.bindConsumer("part.0", input0, properties); + properties.put("partitionIndex", "1"); + QueueChannel input1 = new QueueChannel(); + input1.setBeanName("test.input1S"); + bus.bindConsumer("part.0", input1, properties); + properties.put("partitionIndex", "2"); + QueueChannel input2 = new QueueChannel(); + input2.setBeanName("test.input2S"); + bus.bindConsumer("part.0", input2, properties); + + Message message2 = MessageBuilder.withPayload(new byte[]{2}) + .setHeader(IntegrationMessageHeaderAccessor.CORRELATION_ID, "foo") + .setHeader(IntegrationMessageHeaderAccessor.SEQUENCE_NUMBER, 42) + .setHeader(IntegrationMessageHeaderAccessor.SEQUENCE_SIZE, 43) + .setHeader("xdReplyChannel", "bar") + .build(); + output.send(message2); + output.send(new GenericMessage<>(new byte[]{1})); + output.send(new GenericMessage<>(new byte[]{0})); + + Message receive0 = input0.receive(1000); + assertNotNull(receive0); + Message receive1 = input1.receive(1000); + assertNotNull(receive1); + Message receive2 = input2.receive(1000); + assertNotNull(receive2); + + + assertThat(Arrays.asList( + ((byte[]) receive0.getPayload())[0], + ((byte[]) receive1.getPayload())[0], + ((byte[]) receive2.getPayload())[0]), + containsInAnyOrder((byte)0, (byte)1, (byte)2)); + + bus.unbindConsumers("part.0"); + bus.unbindProducers("part.0"); + } + + @Test + @Override + public void createInboundPubSubBeforeOutboundPubSub() throws Exception { + MessageBus messageBus = getMessageBus(); + DirectChannel moduleOutputChannel = new DirectChannel(); + // Test pub/sub by emulating how StreamPlugin handles taps + DirectChannel tapChannel = new DirectChannel(); + QueueChannel moduleInputChannel = new QueueChannel(); + QueueChannel module2InputChannel = new QueueChannel(); + QueueChannel module3InputChannel = new QueueChannel(); + // Create the tap first + String fooTapName = messageBus.isCapable(MessageBus.Capability.DURABLE_PUBSUB) ? "foo.tap:baz.http" : "tap:baz.http"; + messageBus.bindPubSubConsumer(fooTapName, module2InputChannel, null); + + // Then create the stream + messageBus.bindProducer("baz.0", moduleOutputChannel, null); + messageBus.bindConsumer("baz.0", moduleInputChannel, null); + moduleOutputChannel.addInterceptor(new WireTap(tapChannel)); + messageBus.bindPubSubProducer("tap:baz.http", tapChannel, null); + + // Another new module is using tap as an input channel + String barTapName = messageBus.isCapable(MessageBus.Capability.DURABLE_PUBSUB) ? "bar.tap:baz.http" : "tap:baz.http"; + messageBus.bindPubSubConsumer(barTapName, module3InputChannel, null); + Message message = MessageBuilder.withPayload("foo".getBytes()).setHeader(MessageHeaders.CONTENT_TYPE, "foo/bar").build(); + boolean success = false; + boolean retried = false; + while (!success) { + moduleOutputChannel.send(message); + Message inbound = moduleInputChannel.receive(5000); + assertNotNull(inbound); + assertEquals("foo", new String((byte[])inbound.getPayload())); + Message tapped1 = module2InputChannel.receive(5000); + Message tapped2 = module3InputChannel.receive(5000); + if (tapped1 == null || tapped2 == null) { + // listener may not have started + assertFalse("Failed to receive tap after retry", retried); + retried = true; + continue; + } + success = true; + assertEquals("foo", new String((byte[]) tapped1.getPayload())); + assertNull(tapped1.getHeaders().get(XdHeaders.XD_ORIGINAL_CONTENT_TYPE)); + assertEquals("foo", new String((byte[])tapped2.getPayload())); + } + // delete one tap stream is deleted + messageBus.unbindConsumer(barTapName, module3InputChannel); + Message message2 = MessageBuilder.withPayload("bar".getBytes()).setHeader(MessageHeaders.CONTENT_TYPE, "foo/bar").build(); + moduleOutputChannel.send(message2); + + // other tap still receives messages + Message tapped = module2InputChannel.receive(5000); + assertNotNull(tapped); + + // Removed tap does not + assertNull(module3InputChannel.receive(1000)); + + // when other tap stream is deleted + messageBus.unbindConsumer(fooTapName, module2InputChannel); + // Clean up as StreamPlugin would + messageBus.unbindConsumer("baz.0", moduleInputChannel); + messageBus.unbindProducer("baz.0", moduleOutputChannel); + messageBus.unbindProducers("tap:baz.http"); + assertTrue(getBindings(messageBus).isEmpty()); + } + + @Test + @Override + public void testSendAndReceive() throws Exception { + MessageBus messageBus = getMessageBus(); + DirectChannel moduleOutputChannel = new DirectChannel(); + QueueChannel moduleInputChannel = new QueueChannel(); + messageBus.bindProducer("foo.0", moduleOutputChannel, null); + messageBus.bindConsumer("foo.0", moduleInputChannel, null); + Message message = MessageBuilder.withPayload("foo".getBytes()).build(); + // Let the consumer actually bind to the producer before sending a msg + busBindUnbindLatency(); + moduleOutputChannel.send(message); + Message inbound = moduleInputChannel.receive(5000); + assertNotNull(inbound); + assertEquals("foo", new String((byte[])inbound.getPayload())); + messageBus.unbindProducers("foo.0"); + messageBus.unbindConsumers("foo.0"); + } + + // Ignored, since raw mode does not support headers + @Test + @Override + @Ignore + public void testSendAndReceiveNoOriginalContentType() throws Exception { + + } + + @Test + public void testSendAndReceivePubSub() throws Exception { + MessageBus messageBus = getMessageBus(); + DirectChannel moduleOutputChannel = new DirectChannel(); + // Test pub/sub by emulating how StreamPlugin handles taps + DirectChannel tapChannel = new DirectChannel(); + QueueChannel moduleInputChannel = new QueueChannel(); + QueueChannel module2InputChannel = new QueueChannel(); + QueueChannel module3InputChannel = new QueueChannel(); + messageBus.bindProducer("baz.0", moduleOutputChannel, null); + messageBus.bindConsumer("baz.0", moduleInputChannel, null); + moduleOutputChannel.addInterceptor(new WireTap(tapChannel)); + messageBus.bindPubSubProducer("tap:baz.http", tapChannel, null); + // A new module is using the tap as an input channel + String fooTapName = messageBus.isCapable(MessageBus.Capability.DURABLE_PUBSUB) ? "foo.tap:baz.http" : "tap:baz.http"; + messageBus.bindPubSubConsumer(fooTapName, module2InputChannel, null); + // Another new module is using tap as an input channel + String barTapName = messageBus.isCapable(MessageBus.Capability.DURABLE_PUBSUB) ? "bar.tap:baz.http" : "tap:baz.http"; + messageBus.bindPubSubConsumer(barTapName, module3InputChannel, null); + Message message = MessageBuilder.withPayload("foo".getBytes()).build(); + boolean success = false; + boolean retried = false; + while (!success) { + moduleOutputChannel.send(message); + Message inbound = moduleInputChannel.receive(5000); + assertNotNull(inbound); + assertEquals("foo", new String((byte[])inbound.getPayload())); + + Message tapped1 = module2InputChannel.receive(5000); + Message tapped2 = module3InputChannel.receive(5000); + if (tapped1 == null || tapped2 == null) { + // listener may not have started + assertFalse("Failed to receive tap after retry", retried); + retried = true; + continue; + } + success = true; + assertEquals("foo", new String((byte[])tapped1.getPayload())); + assertEquals("foo", new String((byte[])tapped2.getPayload())); + } + // delete one tap stream is deleted + messageBus.unbindConsumer(barTapName, module3InputChannel); + Message message2 = MessageBuilder.withPayload("bar".getBytes()).build(); + moduleOutputChannel.send(message2); + + // other tap still receives messages + Message tapped = module2InputChannel.receive(5000); + assertNotNull(tapped); + + // Removed tap does not + assertNull(module3InputChannel.receive(1000)); + + // when other tap stream is deleted + messageBus.unbindConsumer(fooTapName, module2InputChannel); + // Clean up as StreamPlugin would + messageBus.unbindConsumer("baz.0", moduleInputChannel); + messageBus.unbindProducer("baz.0", moduleOutputChannel); + messageBus.unbindProducers("tap:baz.http"); + assertTrue(getBindings(messageBus).isEmpty()); + } + +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-kafka/src/test/java/org/springframework/xd/dirt/integration/kafka/EmbeddedZookeeper.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-kafka/src/test/java/org/springframework/xd/dirt/integration/kafka/EmbeddedZookeeper.java new file mode 100644 index 000000000..660e1cd46 --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-kafka/src/test/java/org/springframework/xd/dirt/integration/kafka/EmbeddedZookeeper.java @@ -0,0 +1,108 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package org.springframework.xd.dirt.integration.kafka; + +import java.io.File; +import java.net.InetSocketAddress; + +import kafka.utils.TestUtils$; +import kafka.utils.Utils$; +import org.apache.zookeeper.server.NIOServerCnxnFactory; +import org.apache.zookeeper.server.ZooKeeperServer; + +/** + * A port of kafka.zk.EmbeddedZookeeper, compatible with Zookeeper 3.4 API + * + * @author Marius Bogoevici + */ +public class EmbeddedZookeeper { + + private String connectString; + + private File snapshotDir = TestUtils$.MODULE$.tempDir(); + + private File logDir = TestUtils$.MODULE$.tempDir(); + + private int tickTime = 500; + + private final ZooKeeperServer zookeeper; + + private int port; + + private final NIOServerCnxnFactory factory; + + public EmbeddedZookeeper(String connectString) throws Exception { + this.connectString = connectString; + port = Integer.parseInt(connectString.split(":")[1]); + zookeeper = new ZooKeeperServer(snapshotDir, logDir, tickTime); + factory = new NIOServerCnxnFactory(); + factory.configure(new InetSocketAddress("127.0.0.1", port), 100); + factory.startup(zookeeper); + } + + public String getConnectString() { + return connectString; + } + + public File getSnapshotDir() { + return snapshotDir; + } + + public File getLogDir() { + return logDir; + } + + public int getTickTime() { + return tickTime; + } + + public ZooKeeperServer getZookeeper() { + return zookeeper; + } + + public int getPort() { + return port; + } + + public void shutdown() { + try { + zookeeper.shutdown(); + } + catch (Exception e) { + // ignore exception + } + try { + factory.shutdown(); + } + catch (Exception e) { + // ignore exception + } + try { + Utils$.MODULE$.rm(logDir); + } + catch (Exception e) { + // ignore exception + } + try { + Utils$.MODULE$.rm(snapshotDir); + } + catch (Exception e) { + // ignore exception + } + } +} \ No newline at end of file diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-kafka/src/test/java/org/springframework/xd/dirt/integration/kafka/KafkaTestSupport.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-kafka/src/test/java/org/springframework/xd/dirt/integration/kafka/KafkaTestSupport.java new file mode 100644 index 000000000..88d38942a --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-kafka/src/test/java/org/springframework/xd/dirt/integration/kafka/KafkaTestSupport.java @@ -0,0 +1,167 @@ +/* + * Copyright 2014-2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.xd.dirt.integration.kafka; + + +import java.util.Properties; + +import kafka.server.KafkaConfig; +import kafka.server.KafkaServer; +import kafka.utils.SystemTime$; +import kafka.utils.TestUtils; +import kafka.utils.TestZKUtils; +import kafka.utils.Utils; +import kafka.utils.ZKStringSerializer$; +import kafka.utils.ZkUtils; +import org.I0Itec.zkclient.ZkClient; +import org.I0Itec.zkclient.exception.ZkInterruptedException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.junit.Rule; + +import org.springframework.xd.test.AbstractExternalResourceTestSupport; + +/** + * JUnit {@link Rule} that starts an embedded Kafka server (with an associated Zookeeper) + * + * @author Ilayaperumal Gopinathan + * @author Marius Bogoevici + * @since 1.1 + */ +public class KafkaTestSupport extends AbstractExternalResourceTestSupport { + + private static final Logger log = LoggerFactory.getLogger(KafkaTestSupport.class); + + private static final String XD_KAFKA_TEST_EMBEDDED = "XD_KAFKA_TEST_EMBEDDED"; + + public static final boolean embedded; + + private static final String DEFAULT_ZOOKEEPER_CONNECT = "localhost:2181"; + + private static final String DEFAULT_KAFKA_CONNECT = "localhost:9092"; + + private ZkClient zkClient; + + private EmbeddedZookeeper zookeeper; + + private KafkaServer kafkaServer; + + private Properties brokerConfig = TestUtils.createBrokerConfig(0, TestUtils.choosePort(), false); + + static { + embedded = "true".equals(System.getProperty(XD_KAFKA_TEST_EMBEDDED)); + log.info(String.format("Testing with %s Kafka broker", embedded ? "embedded" : "external")); + } + + public KafkaTestSupport() { + super("KAFKA"); + } + + public String getZkConnectString() { + if (embedded) { + return zookeeper.getConnectString(); + } + else { + return DEFAULT_ZOOKEEPER_CONNECT; + } + } + + public ZkClient getZkClient() { + return this.zkClient; + } + + public String getBrokerAddress() { + if (embedded) { + return kafkaServer.config().hostName() + ":" + kafkaServer.config().port(); + } + else { + return DEFAULT_KAFKA_CONNECT; + } + } + + @Override + protected void obtainResource() throws Exception { + if (embedded) { + log.debug("Starting Zookeeper"); + zookeeper = new EmbeddedZookeeper(TestZKUtils.zookeeperConnect()); + log.debug("Started Zookeeper at " + zookeeper.getConnectString()); + try { + int zkConnectionTimeout = 6000; + int zkSessionTimeout = 6000; + zkClient = new ZkClient(getZkConnectString(), zkSessionTimeout, zkConnectionTimeout, ZKStringSerializer$.MODULE$); + } + catch (Exception e) { + zookeeper.shutdown(); + throw e; + } + try { + log.debug("Creating Kafka server"); + Properties brokerConfigProperties = brokerConfig; + kafkaServer = TestUtils.createServer(new KafkaConfig(brokerConfigProperties), SystemTime$.MODULE$); + log.debug("Created Kafka server at " + kafkaServer.config().hostName() + ":" + kafkaServer.config().port()); + } + catch (Exception e) { + zookeeper.shutdown(); + zkClient.close(); + throw e; + } + } + else { + this.zkClient = new ZkClient(DEFAULT_ZOOKEEPER_CONNECT, 5000, 5000, ZKStringSerializer$.MODULE$); + if (ZkUtils.getAllBrokersInCluster(zkClient).size() == 0) { + throw new RuntimeException("Kafka server not available"); + } + } + } + + @Override + protected void cleanupResource() throws Exception { + if (embedded) { + try { + kafkaServer.shutdown(); + } + catch (Exception e) { + // ignore errors on shutdown + log.error(e.getMessage(), e); + } + try { + Utils.rm(kafkaServer.config().logDirs()); + } + catch (Exception e) { + // ignore errors on shutdown + log.error(e.getMessage(), e); + } + } + try { + zkClient.close(); + } + catch (ZkInterruptedException e) { + // ignore errors on shutdown + log.error(e.getMessage(), e); + } + if (embedded) { + try { + zookeeper.shutdown(); + } + catch (Exception e) { + // ignore errors on shutdown + log.error(e.getMessage(), e); + } + } + } + +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-kafka/src/test/java/org/springframework/xd/dirt/integration/kafka/TestKafkaCluster.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-kafka/src/test/java/org/springframework/xd/dirt/integration/kafka/TestKafkaCluster.java new file mode 100644 index 000000000..1acfcae1f --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-kafka/src/test/java/org/springframework/xd/dirt/integration/kafka/TestKafkaCluster.java @@ -0,0 +1,171 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.xd.dirt.integration.kafka; + +import kafka.admin.AdminUtils; +import kafka.consumer.Consumer; +import kafka.consumer.ConsumerConfig; +import kafka.javaapi.consumer.ConsumerConnector; +import kafka.server.KafkaConfig; +import kafka.server.KafkaServerStartable; + +import kafka.utils.TestUtils; +import org.I0Itec.zkclient.ZkClient; +import org.apache.curator.framework.CuratorFramework; +import org.apache.curator.framework.CuratorFrameworkFactory; +import org.apache.curator.framework.recipes.cache.PathChildrenCache; +import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent; +import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener; +import org.apache.curator.retry.RetryUntilElapsed; +import org.apache.curator.test.TestingServer; + +import org.springframework.util.Assert; +import org.springframework.util.SocketUtils; + +import java.io.IOException; +import java.util.Collections; +import java.util.Properties; + + +/** + * A test Kafka + ZooKeeper pair for testing purposes. + * + * @author Eric Bottard + */ +public class TestKafkaCluster { + + private KafkaServerStartable kafkaServer; + + private TestingServer zkServer; + + public TestKafkaCluster() { + try { + zkServer = new TestingServer(SocketUtils.findAvailableTcpPort()); + } + catch (Exception e) { + throw new IllegalStateException(e); + } + KafkaConfig config = getKafkaConfig(zkServer.getConnectString()); + kafkaServer = new KafkaServerStartable(config); + kafkaServer.startup(); + } + + private static KafkaConfig getKafkaConfig(final String zkConnectString) { + scala.collection.Iterator propsI = + TestUtils.createBrokerConfigs(1, false).iterator(); + assert propsI.hasNext(); + Properties props = propsI.next(); + assert props.containsKey("zookeeper.connect"); + props.put("zookeeper.connect", zkConnectString); + return new KafkaConfig(props); + } + + public String getKafkaBrokerString() { + return String.format("localhost:%d", + kafkaServer.serverConfig().port()); + } + + public void stop() throws IOException { + kafkaServer.shutdown(); + zkServer.stop(); + } + + + /** + * See XD-2293. This is used to reproduce Kafka rebalance issues. + */ + public static void main(String[] args) throws Exception { + TestKafkaCluster cluster = new TestKafkaCluster(); + ZkClient client = new ZkClient(cluster.getZkConnectString(), 10000, 10000, KafkaMessageBus.utf8Serializer); + int partitions = 5; + int replication = 1; + AdminUtils.createTopic(client, "mytopic", partitions, replication, new Properties()); + + Properties props = new Properties(); + props.put("zookeeper.connect", cluster.getZkConnectString()); + props.put("group.id", "foo"); + props.put("rebalance.backoff.ms", "2000"); + props.put("rebalance.max.retries", "2000"); + ConsumerConfig config = new ConsumerConfig(props); + + + CuratorFramework curator = CuratorFrameworkFactory.newClient(cluster.getZkConnectString(), new RetryUntilElapsed(1000, 100)); + curator.start(); + + RebalanceListener listener = null; + for (int i = 0; i < 5; i++) { + System.out.format("%nCreating consumer #%d%n", i + 1); + ConsumerConnector connector = Consumer.createJavaConsumerConnector(config); + connector.createMessageStreams(Collections.singletonMap("mytopic", 1)); + if (i == 0) { + PathChildrenCache cache = new PathChildrenCache(curator, "/consumers/foo/owners/mytopic", true); + listener = new RebalanceListener(5); + cache.getListenable().addListener(listener); + cache.start(PathChildrenCache.StartMode.POST_INITIALIZED_EVENT); + } + + synchronized (listener) { + System.out.println("******** Waiting for rebalance..."); + listener.wait(); + } + + } + + System.out.println(); + + } + + public String getZkConnectString() { + return zkServer.getConnectString(); + } + + private static class RebalanceListener implements PathChildrenCacheListener { + + private int expected; + + private int actual; + + private boolean ready; + + public RebalanceListener(int expected) { + this.expected = expected; + } + + @Override + public synchronized void childEvent(CuratorFramework client, PathChildrenCacheEvent event) throws Exception { + System.out.println(event); + System.out.println(event.getData() != null ? new String(event.getData().getData()) : "no data"); + switch (event.getType()) { + case CHILD_ADDED: + actual++; + if (ready && actual == expected) { + System.out.println("*** Moving on... "); + this.notify(); + } + break; + case CHILD_REMOVED: + actual--; + break; + case INITIALIZED: + Assert.isTrue(actual == expected); + ready = true; + this.notify(); + break; + } + } + } +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-local/pom.xml b/spring-cloud-streams-bindings/spring-cloud-streams-binding-local/pom.xml new file mode 100644 index 000000000..634bded29 --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-local/pom.xml @@ -0,0 +1,31 @@ + + + 4.0.0 + + spring-cloud-streams-binding-local + jar + spring-cloud-streams-binding-local + Local(in memory) binding implementation + + + org.springframework.cloud + spring-cloud-streams-bindings-parent + 1.0.0.BUILD-SNAPSHOT + + + + UTF-8 + + + + + org.springframework.cloud + spring-cloud-streams-binding-spi + + + org.springframework.cloud + spring-cloud-streams-binding-test + + + diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-local/src/main/java/org/springframework/xd/dirt/integration/bus/local/LocalMessageBus.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-local/src/main/java/org/springframework/xd/dirt/integration/bus/local/LocalMessageBus.java new file mode 100644 index 000000000..5390e5277 --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-local/src/main/java/org/springframework/xd/dirt/integration/bus/local/LocalMessageBus.java @@ -0,0 +1,418 @@ +/* + * Copyright 2013-2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.springframework.xd.dirt.integration.bus.local; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +import org.springframework.integration.channel.DirectChannel; +import org.springframework.integration.channel.ExecutorChannel; +import org.springframework.integration.channel.PublishSubscribeChannel; +import org.springframework.integration.channel.QueueChannel; +import org.springframework.integration.config.ConsumerEndpointFactoryBean; +import org.springframework.integration.handler.BridgeHandler; +import org.springframework.integration.scheduling.PollerMetadata; +import org.springframework.integration.support.context.NamedComponent; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageChannel; +import org.springframework.messaging.MessageHandler; +import org.springframework.messaging.MessagingException; +import org.springframework.messaging.PollableChannel; +import org.springframework.messaging.SubscribableChannel; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import org.springframework.util.Assert; +import org.springframework.util.MimeType; +import org.springframework.xd.dirt.integration.bus.AbstractBusPropertiesAccessor; +import org.springframework.xd.dirt.integration.bus.Binding; +import org.springframework.xd.dirt.integration.bus.BusProperties; +import org.springframework.xd.dirt.integration.bus.MessageBusSupport; + +/** + * A simple implementation of {@link org.springframework.xd.dirt.integration.bus.MessageBus} for in-process use. For inbound and outbound, creates a + * {@link DirectChannel} or a {@link QueueChannel} depending on whether the binding is aliased or not then bridges the + * passed {@link MessageChannel} to the channel which is registered in the given application context. If that channel + * does not yet exist, it will be created. + * + * @author David Turanski + * @author Mark Fisher + * @author Gary Russell + * @author Jennifer Hickey + * @author Ilayaperumal Gopinathan + * @since 1.0 + */ +public class LocalMessageBus extends MessageBusSupport { + + private static final int DEFAULT_EXECUTOR_CORE_POOL_SIZE = 0; + + private static final int DEFAULT_EXECUTOR_MAX_POOL_SIZE = 200; + + private static final int DEFAULT_EXECUTOR_QUEUE_SIZE = Integer.MAX_VALUE; + + private static final int DEFAULT_EXECUTOR_KEEPALIVE_SECONDS = 60; + + private static final int DEFAULT_REQ_REPLY_CONCURRENCY = 1; + + protected static final Set CONSUMER_REQUEST_REPLY_PROPERTIES = new SetBuilder() + .addAll(CONSUMER_STANDARD_PROPERTIES) + .add(BusProperties.CONCURRENCY) + .build(); + + private volatile PollerMetadata poller; + + private final Map requestReplyChannels = new HashMap(); + + private final ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); + + private volatile int executorCorePoolSize = DEFAULT_EXECUTOR_CORE_POOL_SIZE; + + private volatile int executorMaxPoolSize = DEFAULT_EXECUTOR_MAX_POOL_SIZE; + + private volatile int executorQueueSize = DEFAULT_EXECUTOR_QUEUE_SIZE; + + private volatile int executorKeepAliveSeconds = DEFAULT_EXECUTOR_KEEPALIVE_SECONDS; + + private volatile int queueSize = Integer.MAX_VALUE; + + private final Map reqRepExecutors = new ConcurrentHashMap<>(); + + /** + * Used to create and customize {@link QueueChannel}s when the binding operation involves aliased names. + */ + private final SharedChannelProvider queueChannelProvider = new SharedChannelProvider( + QueueChannel.class) { + + @Override + protected QueueChannel createSharedChannel(String name) { + QueueChannel queueChannel = new QueueChannel(queueSize); + return queueChannel; + } + }; + + private final SharedChannelProvider pubsubChannelProvider = new SharedChannelProvider( + PublishSubscribeChannel.class) { + + @Override + protected PublishSubscribeChannel createSharedChannel(String name) { + PublishSubscribeChannel publishSubscribeChannel = new PublishSubscribeChannel(executor); + publishSubscribeChannel.setIgnoreFailures(true); + return publishSubscribeChannel; + } + }; + + /** + * Set the poller to use when QueueChannels are used. + */ + public void setPoller(PollerMetadata poller) { + this.poller = poller; + } + + /** + * Set the size of the queue when using {@link QueueChannel}s. + */ + public void setQueueSize(int queueSize) { + this.queueSize = queueSize; + } + + /** + * Set the {@link ThreadPoolTaskExecutor}} core pool size to limit the number of concurrent + * threads. The executor is used for PubSub operations. + * Default: 0 (threads created on demand until maxPoolSize). + * @param executorCorePoolSize the pool size. + */ + public void setExecutorCorePoolSize(int executorCorePoolSize) { + this.executorCorePoolSize = executorCorePoolSize; + } + + /** + * Set the {@link ThreadPoolTaskExecutor}} max pool size to limit the number of concurrent + * threads. The executor is used for PubSub operations. + * Default: 200. + * @param executorMaxPoolSize the pool size. + */ + public void setExecutorMaxPoolSize(int executorMaxPoolSize) { + this.executorMaxPoolSize = executorMaxPoolSize; + } + + /** + * Set the {@link ThreadPoolTaskExecutor}} queue size to limit the number of concurrent + * threads. The executor is used for PubSub operations. + * Default: {@link Integer#MAX_VALUE}. + * @param executorQueueSize the queue size. + */ + public void setExecutorQueueSize(int executorQueueSize) { + this.executorQueueSize = executorQueueSize; + } + + /** + * Set the {@link ThreadPoolTaskExecutor}} keep alive seconds. + * The executor is used for PubSub operations. + * @param executorKeepAliveSeconds the keep alive seconds. + */ + public void setExecutorKeepAliveSeconds(int executorKeepAliveSeconds) { + this.executorKeepAliveSeconds = executorKeepAliveSeconds; + } + + @Override + protected void onInit() { + this.executor.setCorePoolSize(this.executorCorePoolSize); + this.executor.setMaxPoolSize(this.executorMaxPoolSize); + this.executor.setQueueCapacity(this.executorQueueSize); + this.executor.setKeepAliveSeconds(this.executorKeepAliveSeconds); + this.executor.setThreadNamePrefix("xd.localbus-"); + this.executor.initialize(); + } + + /** + * For the local bus we bridge the router "output" channel to a queue channel; the queue + * channel gets the name and the source channel is named 'dynamic.output.to.' + name. + * {@inheritDoc} + */ + @Override + public MessageChannel bindDynamicProducer(String name, Properties properties) { + return doBindDynamicProducer(name, "dynamic.output.to." + name, properties); + } + + /** + * For the local bus we bridge the router "output" channel to a pub/sub channel; the pub/sub + * channel gets the name and the source channel is named 'dynamic.output.to.' + name. + * {@inheritDoc} + */ + @Override + public MessageChannel bindDynamicPubSubProducer(String name, Properties properties) { + return doBindDynamicPubSubProducer(name, "dynamic.output.to." + name, properties); + } + + private SharedChannelProvider getChannelProvider(String name) { + SharedChannelProvider channelProvider = directChannelProvider; + // Use queue channel provider in case of named channels: + // point-to-point type syntax (queue:) and job input channel syntax (job:) + if (name.startsWith(P2P_NAMED_CHANNEL_TYPE_PREFIX) || name.startsWith(JOB_CHANNEL_TYPE_PREFIX)) { + channelProvider = queueChannelProvider; + } + return channelProvider; + } + + /** + * Looks up or creates a DirectChannel with the given name and creates a bridge from that channel to the provided + * channel instance. + */ + @Override + public void bindConsumer(String name, MessageChannel moduleInputChannel, Properties properties) { + validateConsumerProperties(name, properties, CONSUMER_STANDARD_PROPERTIES); + doRegisterConsumer(name, moduleInputChannel, getChannelProvider(name), properties); + } + + @Override + public void bindPubSubConsumer(String name, MessageChannel moduleInputChannel, Properties properties) { + validateConsumerProperties(name, properties, CONSUMER_STANDARD_PROPERTIES); + doRegisterConsumer(name, moduleInputChannel, this.pubsubChannelProvider, properties); + } + + private void doRegisterConsumer(String name, MessageChannel moduleInputChannel, + SharedChannelProvider channelProvider, Properties properties) { + Assert.hasText(name, "a valid name is required to register an inbound channel"); + Assert.notNull(moduleInputChannel, "channel must not be null"); + MessageChannel registeredChannel = channelProvider.lookupOrCreateSharedChannel(name); + bridge(name, registeredChannel, moduleInputChannel, + "inbound." + ((NamedComponent) registeredChannel).getComponentName(), + new LocalBusPropertiesAccessor(properties)); + } + + /** + * Looks up or creates a DirectChannel with the given name and creates a bridge to that channel from the provided + * channel instance. + */ + @Override + public void bindProducer(String name, MessageChannel moduleOutputChannel, Properties properties) { + validateConsumerProperties(name, properties, PRODUCER_STANDARD_PROPERTIES); + doRegisterProducer(name, moduleOutputChannel, getChannelProvider(name), properties); + } + + @Override + public void bindPubSubProducer(String name, MessageChannel moduleOutputChannel, + Properties properties) { + validateConsumerProperties(name, properties, PRODUCER_STANDARD_PROPERTIES); + doRegisterProducer(name, moduleOutputChannel, this.pubsubChannelProvider, properties); + } + + private void doRegisterProducer(String name, MessageChannel moduleOutputChannel, + SharedChannelProvider channelProvider, Properties properties) { + Assert.hasText(name, "a valid name is required to register an outbound channel"); + Assert.notNull(moduleOutputChannel, "channel must not be null"); + MessageChannel registeredChannel = channelProvider.lookupOrCreateSharedChannel(name); + bridge(name, moduleOutputChannel, registeredChannel, + "outbound." + ((NamedComponent) registeredChannel).getComponentName(), + new LocalBusPropertiesAccessor(properties)); + } + + @Override + public void bindRequestor(final String name, MessageChannel requests, final MessageChannel replies, + Properties properties) { + validateConsumerProperties(name, properties, CONSUMER_REQUEST_REPLY_PROPERTIES); + final MessageChannel requestChannel = this.findOrCreateRequestReplyChannel(name, "requestor.", properties); + // TODO: handle Pollable ? + Assert.isInstanceOf(SubscribableChannel.class, requests); + ((SubscribableChannel) requests).subscribe(new MessageHandler() { + + @Override + public void handleMessage(Message message) throws MessagingException { + requestChannel.send(message); + } + }); + + ExecutorChannel replyChannel = this.findOrCreateRequestReplyChannel(name, "replier.", properties); + replyChannel.subscribe(new MessageHandler() { + + @Override + public void handleMessage(Message message) throws MessagingException { + replies.send(message); + } + }); + } + + @Override + public void bindReplier(String name, final MessageChannel requests, MessageChannel replies, + Properties properties) { + validateConsumerProperties(name, properties, CONSUMER_REQUEST_REPLY_PROPERTIES); + SubscribableChannel requestChannel = this.findOrCreateRequestReplyChannel(name, "requestor.", properties); + requestChannel.subscribe(new MessageHandler() { + + @Override + public void handleMessage(Message message) throws MessagingException { + requests.send(message); + } + }); + + // TODO: handle Pollable ? + Assert.isInstanceOf(SubscribableChannel.class, replies); + final SubscribableChannel replyChannel = this.findOrCreateRequestReplyChannel(name, "replier.", properties); + ((SubscribableChannel) replies).subscribe(new MessageHandler() { + + @Override + public void handleMessage(Message message) throws MessagingException { + replyChannel.send(message); + } + }); + } + + private synchronized ExecutorChannel findOrCreateRequestReplyChannel(String name, String prefix, + Properties properties) { + String channelName = prefix + name; + ExecutorChannel channel = this.requestReplyChannels.get(channelName); + if (channel == null) { + ThreadPoolTaskExecutor executor = createRequestReplyExecutor(name, properties); + channel = new ExecutorChannel(executor); + channel.setBeanFactory(getBeanFactory()); + this.requestReplyChannels.put(channelName, channel); + this.reqRepExecutors.put(name, executor); + } + return channel; + } + + private ThreadPoolTaskExecutor createRequestReplyExecutor(String name, Properties properties) { + ThreadPoolTaskExecutor executor = new ThreadPoolTaskExecutor(); + executor.setCorePoolSize(new LocalBusPropertiesAccessor(properties).getConcurrency(DEFAULT_REQ_REPLY_CONCURRENCY)); + executor.setThreadNamePrefix("xd.localBus." + name + "-"); + executor.initialize(); + return executor; + } + + @Override + public void unbindProducer(String name, MessageChannel channel) { + this.requestReplyChannels.remove("replier." + name); + MessageChannel requestChannel = this.requestReplyChannels.remove("requestor." + name); + if (requestChannel == null) { + super.unbindProducer(name, channel); + } + ThreadPoolTaskExecutor executor = this.reqRepExecutors.remove(name); + if (executor != null) { + executor.shutdown(); + } + } + + protected BridgeHandler bridge(String name, MessageChannel from, MessageChannel to, String bridgeName, + LocalBusPropertiesAccessor properties) { + return bridge(name, from, to, bridgeName, null, properties); + } + + + protected BridgeHandler bridge(String name, MessageChannel from, MessageChannel to, String bridgeName, + final Collection acceptedMimeTypes, LocalBusPropertiesAccessor properties) { + + final boolean isInbound = bridgeName.startsWith("inbound."); + + BridgeHandler handler = new BridgeHandler() { + + @Override + protected boolean shouldCopyRequestHeaders() { + return false; + } + + @Override + protected Object handleRequestMessage(Message requestMessage) { + return requestMessage; + } + + }; + + handler.setBeanFactory(getBeanFactory()); + handler.setOutputChannel(to); + handler.setBeanName(bridgeName); + handler.afterPropertiesSet(); + + // Usage of a CEFB allows to handle both Subscribable & Pollable channels the same way + ConsumerEndpointFactoryBean cefb = new ConsumerEndpointFactoryBean(); + cefb.setInputChannel(from); + cefb.setHandler(handler); + cefb.setBeanFactory(getBeanFactory()); + if (from instanceof PollableChannel) { + cefb.setPollerMetadata(poller); + } + try { + cefb.afterPropertiesSet(); + } + catch (Exception e) { + throw new IllegalStateException(e); + } + + try { + cefb.getObject().setComponentName(handler.getComponentName()); + Binding binding = isInbound ? Binding.forConsumer(name, cefb.getObject(), to, properties) + : Binding.forProducer(name, from, cefb.getObject(), properties); + addBinding(binding); + binding.start(); + } + catch (Exception e) { + throw new IllegalStateException(e); + } + return handler; + } + + protected T getBean(String name, Class requiredType) { + return getApplicationContext().getBean(name, requiredType); + } + + private static class LocalBusPropertiesAccessor extends AbstractBusPropertiesAccessor { + + public LocalBusPropertiesAccessor(Properties properties) { + super(properties); + } + + } + +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-local/src/main/resources/META-INF/spring-xd/bus/local-bus.xml b/spring-cloud-streams-bindings/spring-cloud-streams-binding-local/src/main/resources/META-INF/spring-xd/bus/local-bus.xml new file mode 100644 index 000000000..c9aa3c6ba --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-local/src/main/resources/META-INF/spring-xd/bus/local-bus.xml @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-local/src/test/java/org/springframework/xd/dirt/integration/bus/local/LocalMessageBusTests.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-local/src/test/java/org/springframework/xd/dirt/integration/bus/local/LocalMessageBusTests.java new file mode 100644 index 000000000..61b6f7605 --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-local/src/test/java/org/springframework/xd/dirt/integration/bus/local/LocalMessageBusTests.java @@ -0,0 +1,184 @@ +/* + * Copyright 2013-2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.xd.dirt.integration.bus.local; + +import java.util.Collection; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; + +import org.junit.Assert; +import org.junit.Test; + +import org.springframework.context.support.GenericApplicationContext; +import org.springframework.http.MediaType; +import org.springframework.integration.channel.DirectChannel; +import org.springframework.integration.channel.interceptor.WireTap; +import org.springframework.integration.support.DefaultMessageBuilderFactory; +import org.springframework.integration.support.MessageBuilder; +import org.springframework.integration.support.utils.IntegrationUtils; +import org.springframework.integration.test.util.TestUtils; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageHandler; +import org.springframework.messaging.MessageHeaders; +import org.springframework.messaging.MessagingException; +import org.springframework.messaging.support.GenericMessage; +import org.springframework.scheduling.concurrent.ThreadPoolTaskExecutor; +import org.springframework.xd.dirt.integration.bus.AbstractMessageBusTests; +import org.springframework.xd.dirt.integration.bus.MessageBus; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNotSame; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +/** + * @author Gary Russell + * @author David Turanski + * @since 1.0 + */ +public class LocalMessageBusTests extends AbstractMessageBusTests { + + @Override + protected MessageBus getMessageBus() throws Exception { + LocalMessageBus bus = new LocalMessageBus(); + GenericApplicationContext applicationContext = new GenericApplicationContext(); + applicationContext.getBeanFactory().registerSingleton( + IntegrationUtils.INTEGRATION_MESSAGE_BUILDER_FACTORY_BEAN_NAME, + new DefaultMessageBuilderFactory()); + applicationContext.refresh(); + bus.setApplicationContext(applicationContext); + bus.setExecutorCorePoolSize(2); + bus.setExecutorMaxPoolSize(10); + bus.setExecutorKeepAliveSeconds(59); + bus.setExecutorQueueSize(Integer.MAX_VALUE - 1); + bus.afterPropertiesSet(); + return bus; + } + + protected Collection getBindings(MessageBus testMessageBus) { + return getBindingsFromMsgBus(testMessageBus); + } + + @Test + public void testProps() throws Exception { + LocalMessageBus bus = (LocalMessageBus) getMessageBus(); + ThreadPoolTaskExecutor exec = TestUtils.getPropertyValue(bus, "executor", ThreadPoolTaskExecutor.class); + assertEquals(2, exec.getCorePoolSize()); + assertEquals(10, exec.getMaxPoolSize()); + assertEquals(59, exec.getKeepAliveSeconds()); + Assert.assertEquals(Integer.MAX_VALUE - 1, TestUtils.getPropertyValue(exec, "queueCapacity")); + } + + @Test + public void testPayloadConversionNotNeededExplicitType() throws Exception { + LocalMessageBus bus = (LocalMessageBus) getMessageBus(); + verifyPayloadConversion(new TestPayload(), bus); + } + + @Test + public void testNoPayloadConversionByDefault() throws Exception { + LocalMessageBus bus = (LocalMessageBus) getMessageBus(); + verifyPayloadConversion(new TestPayload(), bus); + } + + @Test + public void testTapDoesntHurtStream() throws Exception { + LocalMessageBus bus = (LocalMessageBus) getMessageBus(); + DirectChannel moduleOutputChannel = new DirectChannel(); + moduleOutputChannel.setBeanName("bangOut"); + DirectChannel tapChannel = new DirectChannel(); + tapChannel.setBeanName("tapChannel"); + WireTap tap = new WireTap(tapChannel); + moduleOutputChannel.addInterceptor(tap); + bus.bindProducer("bang.0", moduleOutputChannel, null); + final AtomicBoolean messageReceived = new AtomicBoolean(); + final AtomicReference streamThread = new AtomicReference(); + bus.bindConsumer("bang.0", new DirectChannel() { + + @Override + protected boolean doSend(Message message, long timeout) { + messageReceived.set(true); + streamThread.set(Thread.currentThread()); + return true; + } + }, null); + final CountDownLatch tapped = new CountDownLatch(1); + final AtomicReference tapThread = new AtomicReference(); + bus.bindPubSubProducer("tap:stream:bang.0", tapChannel, null); + bus.bindPubSubConsumer("tap:stream:bang.0", new DirectChannel() { + + @Override + protected boolean doSend(Message message, long timeout) { + tapThread.set(Thread.currentThread()); + tapped.countDown(); + throw new RuntimeException("bang"); + } + }, null); + moduleOutputChannel.send(new GenericMessage("Foo")); + assertTrue(tapped.await(10, TimeUnit.SECONDS)); + assertTrue(messageReceived.get()); + assertSame(Thread.currentThread(), streamThread.get()); + assertNotNull(tapThread.get()); + assertNotSame(Thread.currentThread(), tapThread.get()); + } + + private void verifyPayloadConversion(final Object expectedValue, final LocalMessageBus bus) { + DirectChannel myChannel = new DirectChannel(); + bus.bindConsumer("in", myChannel, null); + DirectChannel input = bus.getBean("in", DirectChannel.class); + assertNotNull(input); + + final AtomicBoolean msgSent = new AtomicBoolean(false); + + myChannel.subscribe(new MessageHandler() { + + @Override + public void handleMessage(Message message) throws MessagingException { + assertEquals(expectedValue, message.getPayload()); + msgSent.set(true); + } + }); + + Message msg = MessageBuilder.withPayload(new TestPayload()) + .setHeader(MessageHeaders.CONTENT_TYPE, MediaType.ALL_VALUE).build(); + + input.send(msg); + assertTrue(msgSent.get()); + } + + static class TestPayload { + + @Override + public String toString() { + return "foo"; + } + + @Override + public boolean equals(Object other) { + return (other instanceof TestPayload && this.toString().equals(other.toString())); + } + + @Override + public int hashCode() { + return this.toString().hashCode(); + } + + } +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/pom.xml b/spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/pom.xml new file mode 100644 index 000000000..ea754bdfe --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/pom.xml @@ -0,0 +1,40 @@ + + + 4.0.0 + + spring-cloud-streams-binding-rabbit + jar + spring-cloud-streams-binding-rabbit + RabbitMQ binding implementation + + + org.springframework.cloud + spring-cloud-streams-bindings-parent + 1.0.0.BUILD-SNAPSHOT + + + + UTF-8 + + + + + org.springframework.cloud + spring-cloud-streams-binding-spi + + + org.springframework.cloud + spring-cloud-streams-binding-test + + + org.springframework.boot + spring-boot-starter-amqp + + + org.springframework.integration + spring-integration-amqp + ${spring-integration.version} + + + diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/main/java/org/springframework/xd/dirt/integration/rabbit/ConnectionFactorySettings.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/main/java/org/springframework/xd/dirt/integration/rabbit/ConnectionFactorySettings.java new file mode 100644 index 000000000..3822e7218 --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/main/java/org/springframework/xd/dirt/integration/rabbit/ConnectionFactorySettings.java @@ -0,0 +1,84 @@ +/* + * + * * Copyright 2011-2015 the original author or authors. + * * + * * Licensed under the Apache License, Version 2.0 (the "License"); + * * you may not use this file except in compliance with the License. + * * You may obtain a copy of the License at + * * + * * http://www.apache.org/licenses/LICENSE-2.0 + * * + * * Unless required by applicable law or agreed to in writing, software + * * distributed under the License is distributed on an "AS IS" BASIS, + * * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * * See the License for the specific language governing permissions and + * * limitations under the License. + * + */ + +package org.springframework.xd.dirt.integration.rabbit; + +import org.springframework.amqp.rabbit.connection.CachingConnectionFactory; +import org.springframework.amqp.rabbit.connection.ConnectionFactory; +import org.springframework.amqp.rabbit.connection.RabbitConnectionFactoryBean; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.boot.autoconfigure.amqp.RabbitProperties; +import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.io.Resource; + +/** + * Configures the connection factory used by the rabbit message bus. + * + * @author Eric Bottard + * @author Gary Russell + */ +@Configuration +public class ConnectionFactorySettings { + + @Value("${spring.rabbitmq.useSSL:false}") + private boolean useSSL; + + @Value("${spring.rabbitmq.sslProperties:}") + private Resource sslPropertiesLocation; + + @Bean + // TODO: Move to spring boot + public ConnectionFactory rabbitConnectionFactory(RabbitProperties config, + com.rabbitmq.client.ConnectionFactory rabbitConnectionFactory) throws Exception { + CachingConnectionFactory factory = new CachingConnectionFactory(rabbitConnectionFactory); + factory.setAddresses(config.getAddresses()); + if (config.getHost() != null) { + factory.setHost(config.getHost()); + factory.setPort(config.getPort()); + } + if (config.getUsername() != null) { + factory.setUsername(config.getUsername()); + } + if (config.getPassword() != null) { + factory.setPassword(config.getPassword()); + } + if (config.getVirtualHost() != null) { + factory.setVirtualHost(config.getVirtualHost()); + } + return factory; + } + + // If no RabbitProperties bean is available, instantiate one, deferring to Spring Boot for populating it + @Configuration + @ConditionalOnMissingBean(RabbitProperties.class) + @EnableConfigurationProperties(RabbitProperties.class) + public static class RabbitPropertiesLoader { + } + + @Bean + public RabbitConnectionFactoryBean rabbitFactory() { + RabbitConnectionFactoryBean rabbitConnectionFactoryBean = new RabbitConnectionFactoryBean(); + rabbitConnectionFactoryBean.setUseSSL(this.useSSL); + rabbitConnectionFactoryBean.setSslPropertiesLocation(this.sslPropertiesLocation); + return rabbitConnectionFactoryBean; + } + +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/main/java/org/springframework/xd/dirt/integration/rabbit/LocalizedQueueConnectionFactory.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/main/java/org/springframework/xd/dirt/integration/rabbit/LocalizedQueueConnectionFactory.java new file mode 100644 index 000000000..fda103988 --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/main/java/org/springframework/xd/dirt/integration/rabbit/LocalizedQueueConnectionFactory.java @@ -0,0 +1,232 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.xd.dirt.integration.rabbit; + +import java.net.URI; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +import org.springframework.amqp.AmqpException; +import org.springframework.amqp.rabbit.connection.CachingConnectionFactory; +import org.springframework.amqp.rabbit.connection.Connection; +import org.springframework.amqp.rabbit.connection.ConnectionFactory; +import org.springframework.amqp.rabbit.connection.ConnectionListener; +import org.springframework.amqp.rabbit.connection.RabbitConnectionFactoryBean; +import org.springframework.amqp.rabbit.connection.RoutingConnectionFactory; +import org.springframework.core.io.Resource; +import org.springframework.util.Assert; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.util.UriComponentsBuilder; +import org.springframework.xd.dirt.integration.bus.RabbitManagementUtils; + + +/** + * A {@link RoutingConnectionFactory} that determines the node on which a queue is located and + * returns a factory that connects directly to that node. + * The RabbitMQ management plugin is called over REST to determine the node and the corresponding + * address for that node is injected into the connection factory. + * A single instance of each connection factory is retained in a cache. + * If the location cannot be determined, the default connection factory is returned. This connection + * factory is typically configured to connect to all the servers in a fail-over mode. + *

{@link #getTargetConnectionFactory(Object)} is invoked by the + * {@code SimpleMessageListenerContainer}, when establishing a connection, with the lookup key having + * the format {@code '[queueName]'}. + *

All {@link ConnectionFactory} methods delegate to the default + * + * @author Gary Russell + * @since 1.2 + */ +public class LocalizedQueueConnectionFactory implements ConnectionFactory, RoutingConnectionFactory { + + private final Log logger = LogFactory.getLog(getClass()); + + private final Map nodeFactories = new HashMap<>(); + + private final ConnectionFactory defaultConnectionFactory; + + private final String[] addresses; + + private final String[] adminAdresses; + + private final String[] nodes; + + private final String vhost; + + private final String username; + + private final String password; + + private final boolean useSSL; + + private final Resource sslPropertiesLocation; + + /** + * + * @param defaultConnectionFactory the fallback connection factory to use if the queue can't be located. + * @param addresses the rabbitmq server addresses (host:port, ...). + * @param adminAddresses the rabbitmq admin addresses (http://host:port, ...) must be the same length + * as addresses. + * @param nodes the rabbitmq nodes corresponding to addresses (rabbit@server1, ...). + * @param vhost the virtual host. + * @param username the user name. + * @param password the password. + */ + public LocalizedQueueConnectionFactory(ConnectionFactory defaultConnectionFactory, + String[] addresses, String[] adminAddresses, String[] nodes, String vhost, + String username, String password, boolean useSSL, Resource sslPropertiesLocation) { + Assert.isTrue(addresses.length == adminAddresses.length + && addresses.length == nodes.length, + "'addresses', 'adminAddresses', and 'nodes' properties must have equal length"); + this.defaultConnectionFactory = defaultConnectionFactory; + this.addresses = Arrays.copyOf(addresses, addresses.length); + this.adminAdresses = Arrays.copyOf(adminAddresses, adminAddresses.length); + this.nodes = Arrays.copyOf(nodes, nodes.length); + this.vhost = vhost; + this.username = username; + this.password = password; + this.useSSL = useSSL; + this.sslPropertiesLocation = sslPropertiesLocation; + } + + @Override + public Connection createConnection() throws AmqpException { + return this.defaultConnectionFactory.createConnection(); + } + + @Override + public String getHost() { + return this.defaultConnectionFactory.getHost(); + } + + @Override + public int getPort() { + return this.defaultConnectionFactory.getPort(); + } + + @Override + public String getVirtualHost() { + return this.vhost; + } + + @Override + public void addConnectionListener(ConnectionListener listener) { + this.defaultConnectionFactory.addConnectionListener(listener); + } + + @Override + public boolean removeConnectionListener(ConnectionListener listener) { + return this.defaultConnectionFactory.removeConnectionListener(listener); + } + + @Override + public void clearConnectionListeners() { + this.defaultConnectionFactory.clearConnectionListeners(); + } + + @Override + public ConnectionFactory getTargetConnectionFactory(Object key) { + String queue = ((String) key); + queue = queue.substring(1, queue.length() - 1); + ConnectionFactory connectionFactory = determineConnectionFactory(queue); + if (connectionFactory == null) { + return this.defaultConnectionFactory; + } + else { + return connectionFactory; + } + } + + private ConnectionFactory determineConnectionFactory(String queue) { + for (int i = 0; i < this.adminAdresses.length; i++) { + String adminUri = this.adminAdresses[i]; + RestTemplate template = createRestTemplate(adminUri); + URI uri = UriComponentsBuilder.fromUriString(adminUri + "/api") + .pathSegment("queues", "{vhost}", "{queue}") + .buildAndExpand(this.vhost, queue).encode().toUri(); + try { + @SuppressWarnings("unchecked") + Map queueInfo = template.getForObject(uri, Map.class); + if (queueInfo != null) { + String node = (String) queueInfo.get("node"); + if (node != null) { + for (int j = 0; j < this.nodes.length; j++) { + if (this.nodes[j].equals(node)) { + return nodeConnectionFactory(queue, j); + } + } + } + } + } + catch (Exception e) { + logger.error("Failed to determine queue location for: " + queue + " at: " + + uri.toString(), e); + } + } + logger.warn("Failed to determine queue location for: " + queue); + return null; + } + + private synchronized ConnectionFactory nodeConnectionFactory(String queue, int index) throws Exception { + String address = this.addresses[index]; + String node = this.nodes[index]; + if (logger.isDebugEnabled()) { + logger.debug("Queue: " + queue + " is on node: " + node + " at: " + address); + } + ConnectionFactory cf = this.nodeFactories.get(node); + if (cf == null) { + if (logger.isDebugEnabled()) { + logger.debug("Creating new connection factory for: " + address); + } + cf = createConnectionFactory(address); + this.nodeFactories.put(node, cf); + } + return cf; + } + + /** + * Create a RestTemplate for the supplied URI. + * @param adminUri the URI. + * @return the template. + */ + protected RestTemplate createRestTemplate(String adminUri) { + return RabbitManagementUtils.buildRestTemplate(adminUri, this.username, this.password); + } + + /** + * Create a dedicated connection factory for the address. + * @param address the address to which the factory should connect. + * @return the connection factory. + * @throws Exception if errors occur during creation. + */ + protected ConnectionFactory createConnectionFactory(String address) throws Exception { + RabbitConnectionFactoryBean rcfb = new RabbitConnectionFactoryBean(); + rcfb.setUseSSL(this.useSSL); + rcfb.setSslPropertiesLocation(this.sslPropertiesLocation); + rcfb.afterPropertiesSet(); + CachingConnectionFactory ccf = new CachingConnectionFactory(rcfb.getObject()); + ccf.setAddresses(address); + ccf.setUsername(this.username); + ccf.setPassword(this.password); + ccf.setVirtualHost(this.vhost); + return ccf; + } + +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/main/java/org/springframework/xd/dirt/integration/rabbit/RabbitBusCleaner.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/main/java/org/springframework/xd/dirt/integration/rabbit/RabbitBusCleaner.java new file mode 100644 index 000000000..c739005cf --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/main/java/org/springframework/xd/dirt/integration/rabbit/RabbitBusCleaner.java @@ -0,0 +1,256 @@ +/* + * Copyright 2015 the original author or authors. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.xd.dirt.integration.rabbit; + +import java.net.URI; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.springframework.web.client.RestTemplate; +import org.springframework.web.util.UriComponentsBuilder; +import org.springframework.xd.dirt.integration.bus.BusCleaner; +import org.springframework.xd.dirt.integration.bus.BusUtils; +import org.springframework.xd.dirt.integration.bus.MessageBusSupport; +import org.springframework.xd.dirt.integration.bus.RabbitAdminException; +import org.springframework.xd.dirt.integration.bus.RabbitManagementUtils; + + +/** + * Implementation of {@link org.springframework.xd.dirt.integration.bus.BusCleaner} for the {@code RabbitMessageBus}. + * @author Gary Russell + * @author David Turanski + * @since 1.2 + */ +public class RabbitBusCleaner implements BusCleaner { + + private final static Logger logger = LoggerFactory.getLogger(RabbitBusCleaner.class); + + @Override + public Map> clean(String entity, boolean isJob) { + return clean("http://localhost:15672", "guest", "guest", "/", "xdbus.", entity, isJob); + } + + public Map> clean(String adminUri, String user, String pw, String vhost, + String busPrefix, String entity, boolean isJob) { + return doClean( + adminUri == null ? "http://localhost:15672" : adminUri, + user == null ? "guest" : user, + pw == null ? "guest" : pw, + vhost == null ? "/" : vhost, + busPrefix == null ? "xdbus." : busPrefix, + entity, isJob); + } + + private Map> doClean(String adminUri, String user, String pw, String vhost, + String busPrefix, String entity, boolean isJob) { + RestTemplate restTemplate = RabbitManagementUtils.buildRestTemplate(adminUri, user, pw); + List removedQueues = isJob + ? null//findJobQueues(adminUri, vhost, busPrefix, entity, restTemplate) + : findStreamQueues(adminUri, vhost, busPrefix, entity, restTemplate); + ExchangeCandidateCallback callback = null; + if (isJob) { +// String pattern; +// if (entity.endsWith("*")) { +// pattern = entity.substring(0, entity.length() - 1) + "[^.]*"; +// } +// else { +// pattern = entity; +// } +// Collection exchangeNames = JobEventsListenerPlugin.getEventListenerChannels(pattern).values(); +// final Set jobExchanges = new HashSet<>(); +// for (String exchange : exchangeNames) { +// jobExchanges.add(Pattern.compile(MessageBusSupport.applyPrefix(busPrefix, +// MessageBusSupport.applyPubSub(exchange)))); +// } +// jobExchanges.add(Pattern.compile(MessageBusSupport.applyPrefix(busPrefix, MessageBusSupport.applyPubSub( +// JobEventsListenerPlugin.getEventListenerChannelName(pattern))))); +// callback = new ExchangeCandidateCallback() { +// +// @Override +// public boolean isCandidate(String exchangeName) { +// for (Pattern pattern : jobExchanges) { +// Matcher matcher = pattern.matcher(exchangeName); +// if (matcher.matches()) { +// return true; +// } +// } +// return false; +// } +// +// }; + } + else { + final String tapPrefix = adjustPrefix(MessageBusSupport.applyPrefix(busPrefix, + MessageBusSupport.applyPubSub(BusUtils.constructTapPrefix(entity)))); + callback = new ExchangeCandidateCallback() { + + @Override + public boolean isCandidate(String exchangeName) { + return exchangeName.startsWith(tapPrefix); + } + }; + } + List removedExchanges = findExchanges(adminUri, vhost, busPrefix, entity, restTemplate, callback); + // Delete the queues in reverse order to enable re-running after a partial success. + // The queue search above starts with 0 and terminates on a not found. + for (int i = removedQueues.size() - 1; i >= 0; i--) { + String queueName = removedQueues.get(i); + URI uri = UriComponentsBuilder.fromUriString(adminUri + "/api") + .pathSegment("queues", "{vhost}", "{stream}") + .buildAndExpand(vhost, queueName).encode().toUri(); + restTemplate.delete(uri); + if (logger.isDebugEnabled()) { + logger.debug("deleted queue: " + queueName); + } + } + Map> results = new HashMap<>(); + if (removedQueues.size() > 0) { + results.put("queues", removedQueues); + } + // Fanout exchanges for taps + for (String exchange : removedExchanges) { + URI uri = UriComponentsBuilder.fromUriString(adminUri + "/api") + .pathSegment("exchanges", "{vhost}", "{name}") + .buildAndExpand(vhost, exchange).encode().toUri(); + restTemplate.delete(uri); + if (logger.isDebugEnabled()) { + logger.debug("deleted exchange: " + exchange); + } + } + if (removedExchanges.size() > 0) { + results.put("exchanges", removedExchanges); + } + return results; + } + + private List findStreamQueues(String adminUri, String vhost, String busPrefix, String stream, + RestTemplate restTemplate) { + String queueNamePrefix = adjustPrefix(MessageBusSupport.applyPrefix(busPrefix, stream)); + List> queues = listAllQueues(adminUri, vhost, restTemplate); + List removedQueues = new ArrayList<>(); + for (Map queue : queues) { + String queueName = (String) queue.get("name"); + if (queueName.startsWith(queueNamePrefix)) { + checkNoConsumers(queueName, queue); + removedQueues.add(queueName); + } + } + return removedQueues; + } + +// private List findJobQueues(String adminUri, String vhost, String busPrefix, String job, +// RestTemplate restTemplate) { +// List removedQueues = new ArrayList<>(); +// String jobQueueName = MessageBusSupport.applyPrefix(busPrefix, +// AbstractJobPlugin.getJobChannelName(job)); +// String jobRequestsQueuePrefix = adjustPrefix(MessageBusSupport.applyPrefix(busPrefix, +// AbstractJobPlugin.getJobChannelName(job))); +// List> queues = listAllQueues(adminUri, vhost, restTemplate); +// for (Map queue : queues) { +// String queueName = (String) queue.get("name"); +// if (job.endsWith("*")) { +// if (queueName.startsWith(jobQueueName.substring(0, jobQueueName.length() - 1))) { +// checkNoConsumers(queueName, queue); +// removedQueues.add(queueName); +// } +// } +// else { +// if (queueName.equals(jobQueueName)) { +// checkNoConsumers(queueName, queue); +// removedQueues.add(queueName); +// } +// else if (queueName.startsWith(jobRequestsQueuePrefix) +// && queueName.endsWith(MessageBusSupport.applyRequests(""))) { +// checkNoConsumers(queueName, queue); +// removedQueues.add(queueName); +// } +// } +// } +// return removedQueues; +// } + + private List> listAllQueues(String adminUri, String vhost, RestTemplate restTemplate) { + URI uri = UriComponentsBuilder.fromUriString(adminUri + "/api") + .pathSegment("queues", "{vhost}") + .buildAndExpand(vhost).encode().toUri(); + @SuppressWarnings("unchecked") + List> queues = restTemplate.getForObject(uri, List.class); + return queues; + } + + private String adjustPrefix(String prefix) { + if (prefix.endsWith("*")) { + return prefix.substring(0, prefix.length() - 1); + } + else { + return prefix + BusUtils.GROUP_INDEX_DELIMITER; + } + } + + private void checkNoConsumers(String queueName, Map queue) { + if (!queue.get("consumers").equals(Integer.valueOf(0))) { + throw new RabbitAdminException("Queue " + queueName + " is in use"); + } + } + + @SuppressWarnings("unchecked") + private List findExchanges(String adminUri, String vhost, String busPrefix, String entity, + RestTemplate restTemplate, ExchangeCandidateCallback callback) { + List removedExchanges = new ArrayList<>(); + URI uri = UriComponentsBuilder.fromUriString(adminUri + "/api") + .pathSegment("exchanges", "{vhost}") + .buildAndExpand(vhost).encode().toUri(); + List> exchanges = restTemplate.getForObject(uri, List.class); + for (Map exchange : exchanges) { + String exchangeName = (String) exchange.get("name"); + if (callback.isCandidate(exchangeName)) { + uri = UriComponentsBuilder.fromUriString(adminUri + "/api") + .pathSegment("exchanges", "{vhost}", "{name}", "bindings", "source") + .buildAndExpand(vhost, exchangeName).encode().toUri(); + List> bindings = restTemplate.getForObject(uri, List.class); + if (bindings.size() == 0) { + uri = UriComponentsBuilder.fromUriString(adminUri + "/api") + .pathSegment("exchanges", "{vhost}", "{name}", "bindings", "destination") + .buildAndExpand(vhost, exchangeName).encode().toUri(); + bindings = restTemplate.getForObject(uri, List.class); + if (bindings.size() == 0) { + removedExchanges.add((String) exchange.get("name")); + } + else { + throw new RabbitAdminException("Cannot delete exchange " + exchangeName + + "; it is a destination: " + bindings); + } + } + else { + throw new RabbitAdminException("Cannot delete exchange " + exchangeName + "; it has bindings: " + + bindings); + } + } + } + return removedExchanges; + } + + private interface ExchangeCandidateCallback { + + boolean isCandidate(String exchangeName); + } + +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/main/java/org/springframework/xd/dirt/integration/rabbit/RabbitMessageBus.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/main/java/org/springframework/xd/dirt/integration/rabbit/RabbitMessageBus.java new file mode 100644 index 000000000..073b43e2e --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/main/java/org/springframework/xd/dirt/integration/rabbit/RabbitMessageBus.java @@ -0,0 +1,1066 @@ +/* + * Copyright 2013-2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.xd.dirt.integration.rabbit; + +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.Map; +import java.util.Properties; +import java.util.Set; + +import org.aopalliance.aop.Advice; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.springframework.amqp.core.AcknowledgeMode; +import org.springframework.amqp.core.BindingBuilder; +import org.springframework.amqp.core.DirectExchange; +import org.springframework.amqp.core.Exchange; +import org.springframework.amqp.core.FanoutExchange; +import org.springframework.amqp.core.MessageDeliveryMode; +import org.springframework.amqp.core.MessagePostProcessor; +import org.springframework.amqp.core.MessageProperties; +import org.springframework.amqp.core.Queue; +import org.springframework.amqp.rabbit.config.RetryInterceptorBuilder; +import org.springframework.amqp.rabbit.connection.ConnectionFactory; +import org.springframework.amqp.rabbit.core.BatchingRabbitTemplate; +import org.springframework.amqp.rabbit.core.ChannelCallback; +import org.springframework.amqp.rabbit.core.RabbitAdmin; +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.amqp.rabbit.core.support.BatchingStrategy; +import org.springframework.amqp.rabbit.core.support.SimpleBatchingStrategy; +import org.springframework.amqp.rabbit.listener.SimpleMessageListenerContainer; +import org.springframework.amqp.rabbit.retry.MessageRecoverer; +import org.springframework.amqp.rabbit.retry.RejectAndDontRequeueRecoverer; +import org.springframework.amqp.rabbit.retry.RepublishMessageRecoverer; +import org.springframework.amqp.rabbit.support.DefaultMessagePropertiesConverter; +import org.springframework.amqp.rabbit.support.MessagePropertiesConverter; +import org.springframework.amqp.support.AmqpHeaders; +import org.springframework.amqp.support.postprocessor.DelegatingDecompressingPostProcessor; +import org.springframework.amqp.support.postprocessor.GZipPostProcessor; +import org.springframework.beans.factory.DisposableBean; +import org.springframework.beans.factory.config.ConfigurableListableBeanFactory; +import org.springframework.beans.factory.support.DefaultListableBeanFactory; +import org.springframework.context.Lifecycle; +import org.springframework.context.support.GenericApplicationContext; +import org.springframework.core.io.Resource; +import org.springframework.core.task.SimpleAsyncTaskExecutor; +import org.springframework.expression.Expression; +import org.springframework.expression.ExpressionParser; +import org.springframework.expression.spel.standard.SpelExpressionParser; +import org.springframework.integration.amqp.inbound.AmqpInboundChannelAdapter; +import org.springframework.integration.amqp.outbound.AmqpOutboundEndpoint; +import org.springframework.integration.amqp.support.DefaultAmqpHeaderMapper; +import org.springframework.integration.channel.DirectChannel; +import org.springframework.integration.context.IntegrationContextUtils; +import org.springframework.integration.endpoint.EventDrivenConsumer; +import org.springframework.integration.handler.AbstractMessageHandler; +import org.springframework.integration.handler.AbstractReplyProducingMessageHandler; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageChannel; +import org.springframework.messaging.MessageHandler; +import org.springframework.messaging.MessageHeaders; +import org.springframework.messaging.SubscribableChannel; +import org.springframework.retry.interceptor.RetryOperationsInterceptor; +import org.springframework.scheduling.TaskScheduler; +import org.springframework.util.Assert; +import org.springframework.util.ClassUtils; +import org.springframework.util.StringUtils; +import org.springframework.xd.dirt.integration.bus.AbstractBusPropertiesAccessor; +import org.springframework.xd.dirt.integration.bus.Binding; +import org.springframework.xd.dirt.integration.bus.BusProperties; +import org.springframework.xd.dirt.integration.bus.BusUtils; +import org.springframework.xd.dirt.integration.bus.MessageBus; +import org.springframework.xd.dirt.integration.bus.MessageBusSupport; +import org.springframework.xd.dirt.integration.bus.MessageValues; +import org.springframework.xd.dirt.integration.bus.serializer.MultiTypeCodec; + +import com.rabbitmq.client.AMQP; +import com.rabbitmq.client.Channel; +import com.rabbitmq.client.Envelope; + +/** + * A {@link MessageBus} implementation backed by RabbitMQ. + * @author Mark Fisher + * @author Gary Russell + * @author Jennifer Hickey + * @author Gunnar Hillert + * @author Ilayaperumal Gopinathan + * @author David Turanski + */ +public class RabbitMessageBus extends MessageBusSupport implements DisposableBean { + + private static final AcknowledgeMode DEFAULT_ACKNOWLEDGE_MODE = AcknowledgeMode.AUTO; + + private static final MessageDeliveryMode DEFAULT_DEFAULT_DELIVERY_MODE = MessageDeliveryMode.PERSISTENT; + + private static final boolean DEFAULT_DEFAULT_REQUEUE_REJECTED = true; + + private static final int DEFAULT_MAX_CONCURRENCY = 1; + + private static final int DEFAULT_PREFETCH_COUNT = 1; + + private static final String DEFAULT_RABBIT_PREFIX = "xdbus."; + + private static final int DEFAULT_TX_SIZE = 1; + + private static final String[] DEFAULT_REQUEST_HEADER_PATTERNS = new String[] { "STANDARD_REQUEST_HEADERS", "*" }; + + private static final String[] DEFAULT_REPLY_HEADER_PATTERNS = new String[] { "STANDARD_REPLY_HEADERS", "*" }; + + private static final String DEAD_LETTER_EXCHANGE = "DLX"; + + private static final Set RABBIT_CONSUMER_PROPERTIES = new HashSet(Arrays.asList(new String[] { + + BusProperties.MAX_CONCURRENCY, + RabbitPropertiesAccessor.ACK_MODE, + RabbitPropertiesAccessor.PREFETCH, + RabbitPropertiesAccessor.PREFIX, + RabbitPropertiesAccessor.REQUEST_HEADER_PATTERNS, + RabbitPropertiesAccessor.REQUEUE, + RabbitPropertiesAccessor.TRANSACTED, + RabbitPropertiesAccessor.TX_SIZE, + RabbitPropertiesAccessor.AUTO_BIND_DLQ, + RabbitPropertiesAccessor.REPUBLISH_TO_DLQ + })); + + /** + * Standard + retry + rabbit consumer properties. + */ + private static final Set SUPPORTED_BASIC_CONSUMER_PROPERTIES = new SetBuilder() + .addAll(CONSUMER_STANDARD_PROPERTIES) + .addAll(CONSUMER_RETRY_PROPERTIES) + .addAll(RABBIT_CONSUMER_PROPERTIES) + .build(); + + private static final Set SUPPORTED_PUBSUB_CONSUMER_PROPERTIES = new SetBuilder() + .addAll(SUPPORTED_BASIC_CONSUMER_PROPERTIES) + .add(BusProperties.DURABLE) + .build(); + + /** + * Basic + concurrency. + */ + private static final Set SUPPORTED_NAMED_CONSUMER_PROPERTIES = new SetBuilder() + .addAll(SUPPORTED_BASIC_CONSUMER_PROPERTIES) + .add(BusProperties.CONCURRENCY) + .build(); + + /** + * Basic + concurrency + partitioning. + */ + private static final Set SUPPORTED_CONSUMER_PROPERTIES = new SetBuilder() + .addAll(SUPPORTED_BASIC_CONSUMER_PROPERTIES) + .add(BusProperties.CONCURRENCY) + .add(BusProperties.PARTITION_INDEX) + .build(); + + /** + * Basic + concurrency + reply headers + delivery mode (reply). + */ + private static final Set SUPPORTED_REPLYING_CONSUMER_PROPERTIES = new SetBuilder() + // request + .addAll(SUPPORTED_BASIC_CONSUMER_PROPERTIES) + .add(BusProperties.CONCURRENCY) + // reply + .add(RabbitPropertiesAccessor.REPLY_HEADER_PATTERNS) + .add(RabbitPropertiesAccessor.DELIVERY_MODE) + .build(); + + /** + * Rabbit producer properties. + */ + private static final Set SUPPORTED_BASIC_PRODUCER_PROPERTIES = new SetBuilder() + .addAll(PRODUCER_STANDARD_PROPERTIES) + .add(RabbitPropertiesAccessor.DELIVERY_MODE) + .add(RabbitPropertiesAccessor.PREFIX) + .add(RabbitPropertiesAccessor.REQUEST_HEADER_PATTERNS) + .add(BusProperties.COMPRESS) + .build(); + + private static final Set SUPPORTED_PUBSUB_PRODUCER_PROPERTIES = new SetBuilder() + .addAll(SUPPORTED_BASIC_PRODUCER_PROPERTIES) + .addAll(PRODUCER_BATCHING_BASIC_PROPERTIES) + .addAll(PRODUCER_BATCHING_ADVANCED_PROPERTIES) + .build(); + + private static final Set SUPPORTED_NAMED_PRODUCER_PROPERTIES = new SetBuilder() + .addAll(SUPPORTED_BASIC_PRODUCER_PROPERTIES) + .addAll(PRODUCER_BATCHING_BASIC_PROPERTIES) + .addAll(PRODUCER_BATCHING_ADVANCED_PROPERTIES) + .build(); + + /** + * Partitioning + rabbit producer properties. + */ + private static final Set SUPPORTED_PRODUCER_PROPERTIES = new SetBuilder() + .addAll(PRODUCER_PARTITIONING_PROPERTIES) + .addAll(SUPPORTED_BASIC_PRODUCER_PROPERTIES) + .add(BusProperties.DIRECT_BINDING_ALLOWED) + .addAll(PRODUCER_BATCHING_BASIC_PROPERTIES) + .addAll(PRODUCER_BATCHING_ADVANCED_PROPERTIES) + .build(); + + /** + * Basic producer + basic consumer + concurrency + reply headers. + */ + private static final Set SUPPORTED_REQUESTING_PRODUCER_PROPERTIES = new SetBuilder() + // request + .addAll(SUPPORTED_BASIC_PRODUCER_PROPERTIES) + // reply + .addAll(SUPPORTED_BASIC_CONSUMER_PROPERTIES) + .add(BusProperties.CONCURRENCY) + .add(RabbitPropertiesAccessor.REPLY_HEADER_PATTERNS) + .build(); + + private static final MessagePropertiesConverter inboundMessagePropertiesConverter = + new DefaultMessagePropertiesConverter() { + + @Override + public MessageProperties toMessageProperties(AMQP.BasicProperties source, Envelope envelope, + String charset) { + MessageProperties properties = super.toMessageProperties(source, envelope, charset); + properties.setDeliveryMode(null); + return properties; + } + }; + + private static final ExpressionParser EXPRESSION_PARSER = new SpelExpressionParser(); + + private final Logger logger = LoggerFactory.getLogger(this.getClass()); + + private final RabbitAdmin rabbitAdmin; + + private final RabbitTemplate rabbitTemplate = new RabbitTemplate(); + + private final GenericApplicationContext autoDeclareContext = new GenericApplicationContext(); + + private ConnectionFactory connectionFactory; + + private MessagePostProcessor decompressingPostProcessor = new DelegatingDecompressingPostProcessor(); + + private MessagePostProcessor compressingPostProcessor = new GZipPostProcessor(); + + // Default RabbitMQ Container properties + + private volatile AcknowledgeMode defaultAcknowledgeMode = DEFAULT_ACKNOWLEDGE_MODE; + + private volatile boolean defaultChannelTransacted; + + private volatile MessageDeliveryMode defaultDefaultDeliveryMode = DEFAULT_DEFAULT_DELIVERY_MODE; + + private volatile boolean defaultDefaultRequeueRejected = DEFAULT_DEFAULT_REQUEUE_REJECTED; + + private volatile int defaultMaxConcurrency = DEFAULT_MAX_CONCURRENCY; + + private volatile int defaultPrefetchCount = DEFAULT_PREFETCH_COUNT; + + private volatile int defaultTxSize = DEFAULT_TX_SIZE; + + private volatile String defaultPrefix = DEFAULT_RABBIT_PREFIX; + + private volatile String[] defaultRequestHeaderPatterns = DEFAULT_REQUEST_HEADER_PATTERNS; + + private volatile String[] defaultReplyHeaderPatterns = DEFAULT_REPLY_HEADER_PATTERNS; + + private volatile boolean defaultAutoBindDLQ = false; + + private volatile boolean defaultRepublishToDLQ = false; + + private volatile String[] addresses; + + private volatile String[] adminAddresses; + + private volatile String[] nodes; + + private String username; + + private String password; + + private String vhost; + + private boolean useSSL; + + private Resource sslPropertiesLocation; + + private volatile boolean clustered; + + public RabbitMessageBus(ConnectionFactory connectionFactory, MultiTypeCodec codec) { + Assert.notNull(connectionFactory, "connectionFactory must not be null"); + Assert.notNull(codec, "codec must not be null"); + this.connectionFactory = connectionFactory; + this.rabbitTemplate.setConnectionFactory(connectionFactory); + this.rabbitTemplate.afterPropertiesSet(); + this.rabbitAdmin = new RabbitAdmin(connectionFactory); + this.autoDeclareContext.refresh(); + this.rabbitAdmin.setApplicationContext(this.autoDeclareContext); + this.rabbitAdmin.afterPropertiesSet(); + this.setCodec(codec); + } + + /** + * Set a {@link MessagePostProcessor} to decompress messages. Defaults to a + * {@link DelegatingDecompressingPostProcessor} with its default delegates. + * @param decompressingPostProcessor the post processor. + */ + public void setDecompressingPostProcessor(MessagePostProcessor decompressingPostProcessor) { + this.decompressingPostProcessor = decompressingPostProcessor; + } + + /** + * Set a {@link org.springframework.amqp.core.MessagePostProcessor} to compress messages. Defaults to a + * {@link org.springframework.amqp.support.postprocessor.GZipPostProcessor}. + * @param compressingPostProcessor the post processor. + */ + public void setCompressingPostProcessor(MessagePostProcessor compressingPostProcessor) { + this.compressingPostProcessor = compressingPostProcessor; + } + + public void setDefaultAcknowledgeMode(AcknowledgeMode defaultAcknowledgeMode) { + Assert.notNull(defaultAcknowledgeMode, "'defaultAcknowledgeMode' cannot be null"); + this.defaultAcknowledgeMode = defaultAcknowledgeMode; + } + + public void setDefaultChannelTransacted(boolean defaultChannelTransacted) { + this.defaultChannelTransacted = defaultChannelTransacted; + } + + public void setDefaultDefaultDeliveryMode(MessageDeliveryMode defaultDefaultDeliveryMode) { + Assert.notNull(defaultDefaultDeliveryMode, "'defaultDeliveryMode' cannot be null"); + this.defaultDefaultDeliveryMode = defaultDefaultDeliveryMode; + } + + public void setDefaultDefaultRequeueRejected(boolean defaultDefaultRequeueRejected) { + this.defaultDefaultRequeueRejected = defaultDefaultRequeueRejected; + } + + /** + * Set the bus's default max consumers; can be overridden by consumer.maxConcurrency. Values less than 'concurrency' + * will be coerced to be equal to concurrency. + * @param defaultMaxConcurrency The default max concurrency. + */ + public void setDefaultMaxConcurrency(int defaultMaxConcurrency) { + this.defaultMaxConcurrency = defaultMaxConcurrency; + } + + public void setDefaultPrefetchCount(int defaultPrefetchCount) { + this.defaultPrefetchCount = defaultPrefetchCount; + } + + public void setDefaultTxSize(int defaultTxSize) { + this.defaultTxSize = defaultTxSize; + } + + public void setDefaultPrefix(String defaultPrefix) { + Assert.notNull(defaultPrefix, "'defaultPrefix' cannot be null"); + this.defaultPrefix = defaultPrefix.trim(); + } + + public void setDefaultRequestHeaderPatterns(String[] defaultRequestHeaderPatterns) { + this.defaultRequestHeaderPatterns = Arrays.copyOf(defaultRequestHeaderPatterns, + defaultRequestHeaderPatterns.length); + } + + public void setDefaultReplyHeaderPatterns(String[] defaultReplyHeaderPatterns) { + this.defaultReplyHeaderPatterns = Arrays.copyOf(defaultReplyHeaderPatterns, defaultReplyHeaderPatterns.length); + } + + public void setDefaultAutoBindDLQ(boolean defaultAutoBindDLQ) { + this.defaultAutoBindDLQ = defaultAutoBindDLQ; + } + + public void setDefaultRepublishToDLQ(boolean defaultRepublishToDLQ) { + this.defaultRepublishToDLQ = defaultRepublishToDLQ; + } + + public void setAddresses(String[] addresses) { + this.addresses = Arrays.copyOf(addresses, addresses.length); + } + + public void setAdminAddresses(String[] adminAddresses) { + this.adminAddresses = Arrays.copyOf(adminAddresses, adminAddresses.length); + } + + public void setNodes(String[] nodes) { + this.nodes = Arrays.copyOf(nodes, nodes.length); + this.clustered = nodes.length > 1; + } + + public void setUsername(String username) { + this.username = username; + } + + public void setPassword(String password) { + this.password = password; + } + + public void setVhost(String vhost) { + this.vhost = vhost; + } + + public void setUseSSL(boolean useSSL) { + this.useSSL = useSSL; + } + + public void setSslPropertiesLocation(Resource sslPropertiesLocation) { + this.sslPropertiesLocation = sslPropertiesLocation; + } + + @Override + protected void onInit() { + super.onInit(); + if (this.clustered) { + Assert.state(this.addresses.length == this.adminAddresses.length + && this.addresses.length == this.nodes.length, + "'addresses', 'adminAddresses', and 'nodes' properties must have equal length"); + this.connectionFactory = new LocalizedQueueConnectionFactory(this.connectionFactory, this.addresses, + this.adminAddresses, this.nodes, this.vhost, this.username, this.password, this.useSSL, + this.sslPropertiesLocation); + } + } + + @Override + public void bindConsumer(final String name, MessageChannel moduleInputChannel, Properties properties) { + if (logger.isInfoEnabled()) { + logger.info("declaring queue for inbound: " + name); + } + if (name.startsWith(P2P_NAMED_CHANNEL_TYPE_PREFIX)) { + validateConsumerProperties(name, properties, SUPPORTED_NAMED_CONSUMER_PROPERTIES); + } + else { + validateConsumerProperties(name, properties, SUPPORTED_CONSUMER_PROPERTIES); + } + RabbitPropertiesAccessor accessor = new RabbitPropertiesAccessor(properties); + String queueName = applyPrefix(accessor.getPrefix(this.defaultPrefix), name); + int partitionIndex = accessor.getPartitionIndex(); + if (partitionIndex >= 0) { + queueName += "-" + partitionIndex; + } + Map args = queueArgs(accessor, queueName); + Queue queue = new Queue(queueName, true, false, false, args); + declareQueueIfNotPresent(queue); + autoBindDLQ(name, accessor); + doRegisterConsumer(name, moduleInputChannel, queue, accessor, false); + bindExistingProducerDirectlyIfPossible(name, moduleInputChannel); + } + + @Override + public void bindPubSubConsumer(String name, MessageChannel moduleInputChannel, Properties properties) { + String exchangeName = BusUtils.removeGroupFromPubSub(name); + if (logger.isInfoEnabled()) { + logger.info("declaring pubsub for inbound: " + name + ", bound to: " + exchangeName); + } + RabbitPropertiesAccessor accessor = new RabbitPropertiesAccessor(properties); + validateConsumerProperties(name, properties, SUPPORTED_PUBSUB_CONSUMER_PROPERTIES); + String prefix = accessor.getPrefix(this.defaultPrefix); + FanoutExchange exchange = new FanoutExchange(applyPrefix(prefix, applyPubSub(exchangeName))); + declareExchangeIfNotPresent(exchange); + Queue queue; + boolean durable = accessor.isDurable(this.defaultDurableSubscription); + String queueName = applyPrefix(prefix, name); + if (durable) { + Map args = queueArgs(accessor, queueName); + queue = new Queue(queueName, true, false, false, args); + } + else { + queue = new Queue(queueName, false, false, true); + } + declareQueueIfNotPresent(queue); + org.springframework.amqp.core.Binding binding = BindingBuilder.bind(queue).to(exchange); + this.rabbitAdmin.declareBinding(binding); + // register with context so they will be redeclared after a connection failure + if (!this.autoDeclareContext.containsBean(applyPubSub(name))) { + this.autoDeclareContext.getBeanFactory().registerSingleton(applyPubSub(name), queue); + } + String bindingBeanName = exchange.getName() + "." + queue.getName() + ".binding"; + if (!this.autoDeclareContext.containsBean(bindingBeanName)) { + this.autoDeclareContext.getBeanFactory().registerSingleton(bindingBeanName, binding); + } + doRegisterConsumer(name, moduleInputChannel, queue, accessor, true); + if (durable) { + autoBindDLQ(name, accessor); + } + } + + private Map queueArgs(RabbitPropertiesAccessor accessor, String queueName) { + Map args = new HashMap<>(); + if (accessor.getAutoBindDLQ(this.defaultAutoBindDLQ)) { + args.put("x-dead-letter-exchange", applyPrefix(accessor.getPrefix(this.defaultPrefix), "DLX")); + args.put("x-dead-letter-routing-key", queueName); + } + return args; + } + + private void doRegisterConsumer(String name, MessageChannel moduleInputChannel, Queue queue, + RabbitPropertiesAccessor properties, boolean isPubSub) { + // Fix for XD-2503 + // Temporarily overrides the thread context classloader with the one where the SimpleMessageListenerContainer + // is defined + // This allows for the proxying that happens while initializing the SimpleMessageListenerContainer to work + // correctly + ClassLoader originalClassloader = Thread.currentThread().getContextClassLoader(); + try { + ClassUtils.overrideThreadContextClassLoader(SimpleMessageListenerContainer.class.getClassLoader()); + SimpleMessageListenerContainer listenerContainer = new SimpleMessageListenerContainer( + this.connectionFactory); + listenerContainer.setAcknowledgeMode(properties.getAcknowledgeMode(this.defaultAcknowledgeMode)); + listenerContainer.setChannelTransacted(properties.getTransacted(this.defaultChannelTransacted)); + listenerContainer.setDefaultRequeueRejected(properties.getRequeueRejected(this + .defaultDefaultRequeueRejected)); + if (!isPubSub) { + int concurrency = properties.getConcurrency(this.defaultConcurrency); + concurrency = concurrency > 0 ? concurrency : 1; + listenerContainer.setConcurrentConsumers(concurrency); + int maxConcurrency = properties.getMaxConcurrency(this.defaultMaxConcurrency); + if (maxConcurrency > concurrency) { + listenerContainer.setMaxConcurrentConsumers(maxConcurrency); + } + } + listenerContainer.setPrefetchCount(properties.getPrefetchCount(this.defaultPrefetchCount)); + listenerContainer.setTxSize(properties.getTxSize(this.defaultTxSize)); + listenerContainer.setTaskExecutor(new SimpleAsyncTaskExecutor(queue.getName() + "-")); + listenerContainer.setQueues(queue); + int maxAttempts = properties.getMaxAttempts(this.defaultMaxAttempts); + if (maxAttempts > 1 || properties.getRepublishToDLQ(this.defaultRepublishToDLQ)) { + RetryOperationsInterceptor retryInterceptor = RetryInterceptorBuilder.stateless() + .maxAttempts(maxAttempts) + .backOffOptions(properties.getBackOffInitialInterval(this.defaultBackOffInitialInterval), + properties.getBackOffMultiplier(this.defaultBackOffMultiplier), + properties.getBackOffMaxInterval(this.defaultBackOffMaxInterval)) + .recoverer(determineRecoverer(name, properties)) + .build(); + listenerContainer.setAdviceChain(new Advice[] { retryInterceptor }); + } + listenerContainer.setAfterReceivePostProcessors(this.decompressingPostProcessor); + listenerContainer.setMessagePropertiesConverter(RabbitMessageBus.inboundMessagePropertiesConverter); + listenerContainer.afterPropertiesSet(); + AmqpInboundChannelAdapter adapter = new AmqpInboundChannelAdapter(listenerContainer); + adapter.setBeanFactory(this.getBeanFactory()); + DirectChannel bridgeToModuleChannel = new DirectChannel(); + bridgeToModuleChannel.setBeanFactory(this.getBeanFactory()); + bridgeToModuleChannel.setBeanName(name + ".bridge"); + adapter.setOutputChannel(bridgeToModuleChannel); + adapter.setBeanName("inbound." + name); + DefaultAmqpHeaderMapper mapper = new DefaultAmqpHeaderMapper(); + mapper.setRequestHeaderNames(properties.getRequestHeaderPattens(this.defaultRequestHeaderPatterns)); + mapper.setReplyHeaderNames(properties.getReplyHeaderPattens(this.defaultReplyHeaderPatterns)); + adapter.setHeaderMapper(mapper); + adapter.afterPropertiesSet(); + Binding consumerBinding = Binding.forConsumer(name, adapter, moduleInputChannel, properties); + addBinding(consumerBinding); + ReceivingHandler convertingBridge = new ReceivingHandler(); + convertingBridge.setOutputChannel(moduleInputChannel); + convertingBridge.setBeanName(name + ".convert.bridge"); + convertingBridge.afterPropertiesSet(); + bridgeToModuleChannel.subscribe(convertingBridge); + consumerBinding.start(); + } + finally { + Thread.currentThread().setContextClassLoader(originalClassloader); + } + } + + private MessageRecoverer determineRecoverer(String name, RabbitPropertiesAccessor properties) { + if (properties.getRepublishToDLQ(this.defaultRepublishToDLQ)) { + RabbitTemplate errorTemplate = new RabbitTemplate(this.connectionFactory); + String prefix = properties.getPrefix(this.defaultPrefix); + RepublishMessageRecoverer republishMessageRecoverer = new RepublishMessageRecoverer(errorTemplate, + deadLetterExchangeName(prefix), + applyPrefix(prefix, name)); + // TODO: Add container id to republished message headers? (Needs AMQP-489). + return republishMessageRecoverer; + } + else { + return new RejectAndDontRequeueRecoverer(); + } + } + + @Override + public void bindProducer(final String name, MessageChannel moduleOutputChannel, + Properties properties) { + Assert.isInstanceOf(SubscribableChannel.class, moduleOutputChannel); + RabbitPropertiesAccessor accessor = new RabbitPropertiesAccessor(properties); + if (name.startsWith(P2P_NAMED_CHANNEL_TYPE_PREFIX)) { + validateProducerProperties(name, properties, SUPPORTED_NAMED_PRODUCER_PROPERTIES); + } + else { + validateProducerProperties(name, properties, SUPPORTED_PRODUCER_PROPERTIES); + } + if (!bindNewProducerDirectlyIfPossible(name, (SubscribableChannel) moduleOutputChannel, accessor)) { + if (logger.isInfoEnabled()) { + logger.info("declaring queue for outbound: " + name); + } + AmqpOutboundEndpoint queue = this.buildOutboundEndpoint(name, accessor, determineRabbitTemplate(accessor)); + doRegisterProducer(name, moduleOutputChannel, queue, accessor); + } + } + + private AmqpOutboundEndpoint buildOutboundEndpoint(final String name, RabbitPropertiesAccessor properties, + RabbitTemplate rabbitTemplate) { + String queueName = applyPrefix(properties.getPrefix(this.defaultPrefix), name); + String partitionKeyExtractorClass = properties.getPartitionKeyExtractorClass(); + Expression partitionKeyExpression = properties.getPartitionKeyExpression(); + AmqpOutboundEndpoint queue = new AmqpOutboundEndpoint(rabbitTemplate); + if (partitionKeyExpression == null && !StringUtils.hasText(partitionKeyExtractorClass)) { + declareQueueIfNotPresent(new Queue(queueName)); + queue.setRoutingKey(queueName); // uses default exchange + } + else { + queue.setExpressionRoutingKey(EXPRESSION_PARSER.parseExpression(buildPartitionRoutingExpression + (queueName))); + // if the stream is partitioned, create one queue for each target partition + for (int i = 0; i < properties.getNextModuleCount(); i++) { + this.rabbitAdmin.declareQueue(new Queue(queueName + "-" + i)); + } + } + configureOutboundHandler(queue, properties); + return queue; + } + + private void configureOutboundHandler(AmqpOutboundEndpoint handler, RabbitPropertiesAccessor properties) { + DefaultAmqpHeaderMapper mapper = new DefaultAmqpHeaderMapper(); + mapper.setRequestHeaderNames(properties.getRequestHeaderPattens(this.defaultRequestHeaderPatterns)); + mapper.setReplyHeaderNames(properties.getReplyHeaderPattens(this.defaultReplyHeaderPatterns)); + handler.setHeaderMapper(mapper); + handler.setDefaultDeliveryMode(properties.getDeliveryMode(this.defaultDefaultDeliveryMode)); + handler.setBeanFactory(this.getBeanFactory()); + handler.afterPropertiesSet(); + } + + @Override + public void bindPubSubProducer(String name, MessageChannel moduleOutputChannel, + Properties properties) { + validateProducerProperties(name, properties, SUPPORTED_PUBSUB_PRODUCER_PROPERTIES); + RabbitPropertiesAccessor accessor = new RabbitPropertiesAccessor(properties); + String exchangeName = applyPrefix(accessor.getPrefix(this.defaultPrefix), applyPubSub(name)); + declareExchangeIfNotPresent(new FanoutExchange(exchangeName)); + AmqpOutboundEndpoint fanout = new AmqpOutboundEndpoint(determineRabbitTemplate(accessor)); + fanout.setExchangeName(exchangeName); + configureOutboundHandler(fanout, accessor); + doRegisterProducer(name, moduleOutputChannel, fanout, accessor); + } + + private RabbitTemplate determineRabbitTemplate(RabbitPropertiesAccessor properties) { + RabbitTemplate rabbitTemplate = null; + if (properties.isBatchingEnabled(this.defaultBatchingEnabled)) { + BatchingStrategy batchingStrategy = new SimpleBatchingStrategy( + properties.getBatchSize(this.defaultBatchSize), + properties.geteBatchBufferLimit(this.defaultBatchBufferLimit), + properties.getBatchTimeout(this.defaultBatchTimeout)); + rabbitTemplate = new BatchingRabbitTemplate(batchingStrategy, + getApplicationContext().getBean(IntegrationContextUtils.TASK_SCHEDULER_BEAN_NAME, + TaskScheduler.class)); + rabbitTemplate.setConnectionFactory(this.connectionFactory); + } + if (properties.isCompress(this.defaultCompress)) { + if (rabbitTemplate == null) { + rabbitTemplate = new RabbitTemplate(this.connectionFactory); + } + rabbitTemplate.setBeforePublishPostProcessors(this.compressingPostProcessor); + rabbitTemplate.afterPropertiesSet(); + } + if (rabbitTemplate == null) { + rabbitTemplate = this.rabbitTemplate; + } + return rabbitTemplate; + } + + private void doRegisterProducer(final String name, MessageChannel moduleOutputChannel, + AmqpOutboundEndpoint delegate, RabbitPropertiesAccessor properties) { + this.doRegisterProducer(name, moduleOutputChannel, delegate, null, properties); + } + + private void doRegisterProducer(final String name, MessageChannel moduleOutputChannel, + AmqpOutboundEndpoint delegate, String replyTo, RabbitPropertiesAccessor properties) { + Assert.isInstanceOf(SubscribableChannel.class, moduleOutputChannel); + MessageHandler handler = new SendingHandler(delegate, replyTo, properties); + EventDrivenConsumer consumer = new EventDrivenConsumer((SubscribableChannel) moduleOutputChannel, handler); + consumer.setBeanFactory(getBeanFactory()); + consumer.setBeanName("outbound." + name); + consumer.afterPropertiesSet(); + Binding producerBinding = Binding.forProducer(name, moduleOutputChannel, consumer, properties); + addBinding(producerBinding); + producerBinding.start(); + } + + @Override + public void bindRequestor(String name, MessageChannel requests, MessageChannel replies, + Properties properties) { + if (logger.isInfoEnabled()) { + logger.info("binding requestor: " + name); + } + validateProducerProperties(name, properties, SUPPORTED_REQUESTING_PRODUCER_PROPERTIES); + Assert.isInstanceOf(SubscribableChannel.class, requests); + RabbitPropertiesAccessor accessor = new RabbitPropertiesAccessor(properties); + String queueName = applyRequests(name); + AmqpOutboundEndpoint queue = this.buildOutboundEndpoint(queueName, accessor, this.rabbitTemplate); + queue.setBeanFactory(this.getBeanFactory()); + + String replyQueueName = accessor.getPrefix(this.defaultPrefix) + name + ".replies." + + this.getIdGenerator().generateId(); + this.doRegisterProducer(name, requests, queue, replyQueueName, accessor); + Queue replyQueue = new Queue(replyQueueName, false, false, true); // auto-delete + declareQueueIfNotPresent(replyQueue); + // register with context so it will be redeclared after a connection failure + if (!this.autoDeclareContext.containsBean(replyQueueName)) { + this.autoDeclareContext.getBeanFactory().registerSingleton(replyQueueName, replyQueue); + } + this.doRegisterConsumer(name, replies, replyQueue, accessor, false); + } + + @Override + public void bindReplier(String name, MessageChannel requests, MessageChannel replies, + Properties properties) { + if (logger.isInfoEnabled()) { + logger.info("binding replier: " + name); + } + validateConsumerProperties(name, properties, SUPPORTED_REPLYING_CONSUMER_PROPERTIES); + RabbitPropertiesAccessor accessor = new RabbitPropertiesAccessor(properties); + Queue requestQueue = new Queue(applyPrefix(accessor.getPrefix(this.defaultPrefix), applyRequests(name))); + declareQueueIfNotPresent(requestQueue); + this.doRegisterConsumer(name, requests, requestQueue, accessor, false); + + AmqpOutboundEndpoint replyQueue = new AmqpOutboundEndpoint(rabbitTemplate); + replyQueue.setExpressionRoutingKey(EXPRESSION_PARSER.parseExpression("headers['" + AmqpHeaders.REPLY_TO + + "']")); + configureOutboundHandler(replyQueue, accessor); + doRegisterProducer(name, replies, replyQueue, accessor); + } + + /** + * Try passive declaration first, in case the user has pre-configured the queue with incompatible arguments. + * @param queue The queue. + */ + private void declareQueueIfNotPresent(Queue queue) { + if (this.rabbitAdmin.getQueueProperties(queue.getName()) == null) { + this.rabbitAdmin.declareQueue(queue); + } + } + + /** + * Try passive declaration first, in case the user has pre-configured the exchange with incompatible arguments. + * @param exchange + */ + private void declareExchangeIfNotPresent(final Exchange exchange) { + this.rabbitTemplate.execute(new ChannelCallback() { + + @Override + public Void doInRabbit(Channel channel) throws Exception { + try { + channel.exchangeDeclarePassive(exchange.getName()); + } + catch (IOException e) { + RabbitMessageBus.this.rabbitAdmin.declareExchange(exchange); + } + return null; + } + + }); + } + + /** + * If so requested, declare the DLX/DLQ and bind it. The DLQ is bound to the DLX with a routing key of the original + * queue name because we use default exchange routing by queue name for the original message. + * @param name The name. + * @param properties The properties accessor. + */ + private void autoBindDLQ(final String name, RabbitPropertiesAccessor properties) { + if (logger.isDebugEnabled()) { + logger.debug("autoBindDLQ=" + properties.getAutoBindDLQ(this.defaultAutoBindDLQ) + + " for: " + name); + } + if (properties.getAutoBindDLQ(this.defaultAutoBindDLQ)) { + String prefix = properties.getPrefix(this.defaultPrefix); + String queueName = applyPrefix(prefix, name); + String dlqName = constructDLQName(queueName); + Queue dlq = new Queue(dlqName); + declareQueueIfNotPresent(dlq); + final String dlxName = deadLetterExchangeName(prefix); + final DirectExchange dlx = new DirectExchange(dlxName); + declareExchangeIfNotPresent(dlx); + this.rabbitAdmin.declareBinding(BindingBuilder.bind(dlq).to(dlx).with(queueName)); + } + } + + private String deadLetterExchangeName(String prefix) { + return prefix + DEAD_LETTER_EXCHANGE; + } + + @Override + public void unbindConsumer(String name, MessageChannel channel) { + super.unbindConsumer(name, channel); + cleanAutoDeclareContext(name); + } + + @Override + public void unbindConsumers(String name) { + super.unbindConsumers(name); + cleanAutoDeclareContext(name); + } + + private void cleanAutoDeclareContext(String name) { + if (this.autoDeclareContext.containsBean(applyPubSub(name))) { + ConfigurableListableBeanFactory beanFactory = this.autoDeclareContext.getBeanFactory(); + if (beanFactory instanceof DefaultListableBeanFactory) { + ((DefaultListableBeanFactory) beanFactory).destroySingleton(applyPubSub(name)); + } + } + } + + @Override + public boolean isCapable(Capability capability) { + switch (capability) { + case DURABLE_PUBSUB: + return true; + default: + return false; + } + } + + @Override + public void destroy() { + stopBindings(); + } + + @Override + public void doManualAck(LinkedList messageHeadersList) { + Iterator iterator = messageHeadersList.iterator(); + Map channelsToAck = new HashMap(); + while (iterator.hasNext()) { + MessageHeaders messageHeaders = iterator.next(); + if (messageHeaders.containsKey(AmqpHeaders.CHANNEL)) { + Channel channel = (com.rabbitmq.client.Channel) messageHeaders.get(AmqpHeaders.CHANNEL); + Long deliveryTag = (Long) messageHeaders.get(AmqpHeaders.DELIVERY_TAG); + channelsToAck.put(channel, deliveryTag); + } + } + for (Map.Entry entry : channelsToAck.entrySet()) { + try { + ((Channel) entry.getKey()).basicAck(entry.getValue(), true); + } + catch (IOException e) { + logger.error("Exception while manually acknowledging " + e); + } + } + } + + private class SendingHandler extends AbstractMessageHandler implements Lifecycle { + + private final MessageHandler delegate; + + private final String replyTo; + + private final PartitioningMetadata partitioningMetadata; + + private SendingHandler(MessageHandler delegate, String replyTo, RabbitPropertiesAccessor properties) { + this.delegate = delegate; + this.replyTo = replyTo; + this.partitioningMetadata = new PartitioningMetadata(properties, properties.getNextModuleCount()); + this.setBeanFactory(RabbitMessageBus.this.getBeanFactory()); + } + + @Override + protected void handleMessageInternal(Message message) throws Exception { + MessageValues messageToSend = serializePayloadIfNecessary(message); + + if (replyTo != null) { + messageToSend.put(AmqpHeaders.REPLY_TO, this.replyTo); + } + if (this.partitioningMetadata.isPartitionedModule()) { + messageToSend.put(PARTITION_HEADER, determinePartition(message, this.partitioningMetadata)); + } + + this.delegate.handleMessage(messageToSend.toMessage(getMessageBuilderFactory())); + } + + @Override + public void start() { + if (this.delegate instanceof Lifecycle) { + ((Lifecycle) this.delegate).start(); + } + } + + @Override + public void stop() { + if (this.delegate instanceof Lifecycle) { + ((Lifecycle) this.delegate).stop(); + } + } + + @Override + public boolean isRunning() { + if (this.delegate instanceof Lifecycle) { + return ((Lifecycle) this.delegate).isRunning(); + } + else { + return true; + } + } + + } + + private class ReceivingHandler extends AbstractReplyProducingMessageHandler { + + public ReceivingHandler() { + super(); + this.setBeanFactory(RabbitMessageBus.this.getBeanFactory()); + } + + @Override + protected Object handleRequestMessage(Message requestMessage) { + return deserializePayloadIfNecessary(requestMessage).toMessage(getMessageBuilderFactory()); + } + + @Override + protected boolean shouldCopyRequestHeaders() { + /* + * we've already copied the headers so no need for the ARPMH to do it, and we don't want the content-type + * restored if absent. + */ + return false; + } + + } + + /** + * Property accessor for the RabbitMessageBus. Refer to the Spring-AMQP documentation for information on the + * specific properties. + */ + private static class RabbitPropertiesAccessor extends AbstractBusPropertiesAccessor { + + /** + * The acknowledge mode. + */ + private static final String ACK_MODE = "ackMode"; + + /** + * The delivery mode. + */ + private static final String DELIVERY_MODE = "deliveryMode"; + + /** + * The prefetch count (basic qos). + */ + private static final String PREFETCH = "prefetch"; + + /** + * The prefix for queues, exchanges. + */ + private static final String PREFIX = "prefix"; + + /** + * The reply header patterns. + */ + private static final String REPLY_HEADER_PATTERNS = "replyHeaderPatterns"; + + /** + * The request header patterns. + */ + private static final String REQUEST_HEADER_PATTERNS = "requestHeaderPatterns"; + + /** + * Whether delivery failures should be requeued. + */ + private static final String REQUEUE = "requeue"; + + /** + * Whether to use transacted channels. + */ + private static final String TRANSACTED = "transacted"; + + /** + * The number of deliveries between acks. + */ + private static final String TX_SIZE = "txSize"; + + /** + * Whether to automatically declare the DLQ and bind it to the bus DLX. + */ + private static final String AUTO_BIND_DLQ = "autoBindDLQ"; + + /** + * Whether to automatically declare the DLQ and bind it to the bus DLX. + */ + private static final String REPUBLISH_TO_DLQ = "republishToDLQ"; + + public RabbitPropertiesAccessor(Properties properties) { + super(properties); + } + + public AcknowledgeMode getAcknowledgeMode(AcknowledgeMode defaultValue) { + String ackknowledgeMode = getProperty(ACK_MODE); + if (StringUtils.hasText(ackknowledgeMode)) { + return AcknowledgeMode.valueOf(ackknowledgeMode); + } + else { + return defaultValue; + } + } + + public MessageDeliveryMode getDeliveryMode(MessageDeliveryMode defaultValue) { + String deliveryMode = getProperty(DELIVERY_MODE); + if (StringUtils.hasText(deliveryMode)) { + return MessageDeliveryMode.valueOf(deliveryMode); + } + else { + return defaultValue; + } + } + + public int getPrefetchCount(int defaultValue) { + return getProperty(PREFETCH, defaultValue); + } + + public String getPrefix(String defaultValue) { + return getProperty(PREFIX, defaultValue); + } + + public String[] getReplyHeaderPattens(String[] defaultValue) { + return asStringArray(getProperty(REPLY_HEADER_PATTERNS), defaultValue); + } + + public String[] getRequestHeaderPattens(String[] defaultValue) { + return asStringArray(getProperty(REQUEST_HEADER_PATTERNS), defaultValue); + } + + public boolean getRequeueRejected(boolean defaultValue) { + return getProperty(REQUEUE, defaultValue); + } + + public boolean getTransacted(boolean defaultValue) { + return getProperty(TRANSACTED, defaultValue); + } + + public int getTxSize(int defaultValue) { + return getProperty(TX_SIZE, defaultValue); + } + + public boolean getAutoBindDLQ(boolean defaultValue) { + return getProperty(AUTO_BIND_DLQ, defaultValue); + } + + public boolean getRepublishToDLQ(boolean defaultValue) { + return getProperty(REPUBLISH_TO_DLQ, defaultValue); + } + + } + +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/main/java/org/springframework/xd/dirt/integration/rabbit/package-info.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/main/java/org/springframework/xd/dirt/integration/rabbit/package-info.java new file mode 100644 index 000000000..9ee2f84ac --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/main/java/org/springframework/xd/dirt/integration/rabbit/package-info.java @@ -0,0 +1,5 @@ +/** + * This package contains an implementation of the {@link org.springframework.xd.dirt.integration.bus.MessageBus} for RabbitMQ. + */ + +package org.springframework.xd.dirt.integration.rabbit; \ No newline at end of file diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/main/resources/META-INF/spring-xd/bus/rabbit-bus.xml b/spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/main/resources/META-INF/spring-xd/bus/rabbit-bus.xml new file mode 100644 index 000000000..ad188418f --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/main/resources/META-INF/spring-xd/bus/rabbit-bus.xml @@ -0,0 +1,56 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/test/java/org/springframework/xd/dirt/integration/rabbit/LocalizedQueueConnectionFactoryIntegrationTests.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/test/java/org/springframework/xd/dirt/integration/rabbit/LocalizedQueueConnectionFactoryIntegrationTests.java new file mode 100644 index 000000000..2f80f7f5f --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/test/java/org/springframework/xd/dirt/integration/rabbit/LocalizedQueueConnectionFactoryIntegrationTests.java @@ -0,0 +1,69 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.xd.dirt.integration.rabbit; + +import static org.junit.Assert.assertEquals; + +import java.util.UUID; + +import org.junit.Before; +import org.junit.ClassRule; +import org.junit.Test; + +import org.springframework.amqp.core.Queue; +import org.springframework.amqp.rabbit.connection.CachingConnectionFactory; +import org.springframework.amqp.rabbit.connection.ConnectionFactory; +import org.springframework.amqp.rabbit.core.RabbitAdmin; +import org.springframework.amqp.rabbit.core.RabbitTemplate; + + +/** + * + * @author Gary Russell + */ +public class LocalizedQueueConnectionFactoryIntegrationTests { + + @ClassRule + public static RabbitTestSupport rabbitAvailableRule = new RabbitTestSupport(true); + + private LocalizedQueueConnectionFactory lqcf; + + @Before + public void setup() { + ConnectionFactory defaultConnectionFactory = new CachingConnectionFactory("localhost"); + String[] addresses = new String[] { "localhost:9999", "localhost:5672" }; + String[] adminAddresses = new String[] { "http://localhost:15672", "http://localhost:15672" }; + String[] nodes = new String[] { "foo@bar", "rabbit@localhost" }; + String vhost = "/"; + String username = "guest"; + String password = "guest"; + this.lqcf = new LocalizedQueueConnectionFactory(defaultConnectionFactory, addresses, + adminAddresses, nodes, vhost, username, password, false, null); + } + + @Test + public void testConnect() { + RabbitAdmin admin = new RabbitAdmin(this.lqcf); + Queue queue = new Queue(UUID.randomUUID().toString(), false, false, true); + admin.declareQueue(queue); + ConnectionFactory targetConnectionFactory = this.lqcf.getTargetConnectionFactory("[" + queue.getName() + "]"); + RabbitTemplate template = new RabbitTemplate(targetConnectionFactory); + template.convertAndSend("", queue.getName(), "foo"); + assertEquals("foo", template.receiveAndConvert(queue.getName())); + } + +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/test/java/org/springframework/xd/dirt/integration/rabbit/LocalizedQueueConnectionFactoryTests.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/test/java/org/springframework/xd/dirt/integration/rabbit/LocalizedQueueConnectionFactoryTests.java new file mode 100644 index 000000000..d7b022f45 --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/test/java/org/springframework/xd/dirt/integration/rabbit/LocalizedQueueConnectionFactoryTests.java @@ -0,0 +1,186 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.xd.dirt.integration.rabbit; + +import static org.junit.Assert.assertTrue; +import static org.mockito.Matchers.any; +import static org.mockito.Matchers.anyBoolean; +import static org.mockito.Matchers.anyMap; +import static org.mockito.Matchers.anyString; +import static org.mockito.Mockito.atLeast; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; +import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo; +import static org.springframework.test.web.client.response.MockRestResponseCreators.withSuccess; + +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.commons.logging.Log; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Matchers; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +import org.springframework.amqp.rabbit.connection.Connection; +import org.springframework.amqp.rabbit.connection.ConnectionFactory; +import org.springframework.amqp.rabbit.listener.SimpleMessageListenerContainer; +import org.springframework.amqp.utils.test.TestUtils; +import org.springframework.beans.DirectFieldAccessor; +import org.springframework.http.MediaType; +import org.springframework.test.web.client.MockRestServiceServer; +import org.springframework.web.client.RestTemplate; + +import com.rabbitmq.client.Channel; +import com.rabbitmq.client.Consumer; + + +/** + * + * @author Gary Russell + */ +public class LocalizedQueueConnectionFactoryTests { + + private final Map cfs = new HashMap<>(); + + private final Map connections = new HashMap<>(); + + private final Map channels = new HashMap<>(); + + private final Map consumers = new HashMap<>(); + + private final Map consumerTags = new HashMap<>(); + + private final CountDownLatch latch = new CountDownLatch(2); + + @SuppressWarnings("unchecked") + @Test + public void testFailOver() throws Exception { + ConnectionFactory defaultConnectionFactory = mockCF("localhost:1234"); + String rabbit1 = "localhost:1235"; + String rabbit2 = "localhost:1236"; + String[] addresses = new String[] { rabbit1, rabbit2 }; + String[] adminAddresses = new String[] { "http://localhost:11235", "http://localhost:11236" }; + String[] nodes = new String[] { "rabbit@foo", "rabbit@bar" }; + String vhost = "/"; + String username = "guest"; + String password = "guest"; + final AtomicBoolean firstServer = new AtomicBoolean(true); + LocalizedQueueConnectionFactory lqcf = new LocalizedQueueConnectionFactory(defaultConnectionFactory, addresses, + adminAddresses, nodes, vhost, username, password, false, null) { + + private final String[] nodes = new String[] { "rabbit@foo", "rabbit@bar" }; + + @Override + protected RestTemplate createRestTemplate(String adminUri) { + return doCreateRestTemplate(adminUri, firstServer.get() ? nodes[0] : nodes[1]); + } + + @Override + protected ConnectionFactory createConnectionFactory(String address) throws Exception { + return mockCF(address); + } + + }; + Log logger = spy(TestUtils.getPropertyValue(lqcf, "logger", Log.class)); + new DirectFieldAccessor(lqcf).setPropertyValue("logger", logger); + when(logger.isDebugEnabled()).thenReturn(true); + ArgumentCaptor captor = ArgumentCaptor.forClass(String.class); + SimpleMessageListenerContainer container = new SimpleMessageListenerContainer(lqcf); + container.setQueueNames("q"); + container.afterPropertiesSet(); + container.start(); + Channel channel = this.channels.get(rabbit1); + verify(channel).basicConsume(anyString(), anyBoolean(), anyString(), anyBoolean(), + anyBoolean(), anyMap(), + Matchers.any(Consumer.class)); + verify(logger, atLeast(1)).debug(captor.capture()); + assertTrue(assertLog(captor.getAllValues(), "Queue: q is on node: rabbit@foo at: localhost:1235")); + + // Fail rabbit1 and verify the container switches to rabbit2 + + firstServer.set(false); + when(channel.isOpen()).thenReturn(false); + when(this.connections.get(rabbit1).isOpen()).thenReturn(false); + this.consumers.get(rabbit1).handleCancel(consumerTags.get(rabbit1)); + assertTrue(latch.await(10, TimeUnit.SECONDS)); + channel = this.channels.get(rabbit2); + verify(channel).basicConsume(anyString(), anyBoolean(), anyString(), anyBoolean(), + anyBoolean(), anyMap(), + Matchers.any(Consumer.class)); + container.stop(); + verify(logger, atLeast(1)).debug(captor.capture()); + assertTrue(assertLog(captor.getAllValues(), "Queue: q is on node: rabbit@bar at: localhost:1236")); + } + + private boolean assertLog(List logRows, String expected) { + for (String log : logRows) { + if (log.contains(expected)) { + return true; + } + } + return false; + } + + private RestTemplate doCreateRestTemplate(String uri, String node) { + RestTemplate template = new RestTemplate(); + MockRestServiceServer server = MockRestServiceServer.createServer(template); + server.expect(requestTo(uri + "/api/queues/%2F/q")) + .andRespond(withSuccess("{ \"node\" : \"" + + node + + "\" }", MediaType.APPLICATION_JSON)); + return template; + } + + @SuppressWarnings("unchecked") + private ConnectionFactory mockCF(final String address) throws Exception { + ConnectionFactory connectionFactory = mock(ConnectionFactory.class); + Connection connection = mock(Connection.class); + Channel channel = mock(Channel.class); + when(connectionFactory.createConnection()).thenReturn(connection); + when(connection.createChannel(false)).thenReturn(channel); + when(connection.isOpen()).thenReturn(true); + when(channel.isOpen()).thenReturn(true); + doAnswer(new Answer() { + + @Override + public String answer(InvocationOnMock invocation) throws Throwable { + String tag = UUID.randomUUID().toString(); + consumers.put(address, (Consumer) invocation.getArguments()[6]); + consumerTags.put(address, tag); + latch.countDown(); + return tag; + } + }).when(channel).basicConsume(anyString(), anyBoolean(), anyString(), anyBoolean(), anyBoolean(), anyMap(), + any(Consumer.class)); + when(connectionFactory.getHost()).thenReturn(address); + this.cfs.put(address, connectionFactory); + this.connections.put(address, connection); + this.channels.put(address, channel); + return connectionFactory; + } + +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/test/java/org/springframework/xd/dirt/integration/rabbit/RabbitAdminTestSupport.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/test/java/org/springframework/xd/dirt/integration/rabbit/RabbitAdminTestSupport.java new file mode 100644 index 000000000..2dd215235 --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/test/java/org/springframework/xd/dirt/integration/rabbit/RabbitAdminTestSupport.java @@ -0,0 +1,57 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.xd.dirt.integration.rabbit; + + +import org.springframework.http.HttpStatus; +import org.springframework.web.client.HttpClientErrorException; +import org.springframework.web.client.RestTemplate; +import org.springframework.xd.test.AbstractExternalResourceTestSupport; + +import java.util.Map; + +/** + * JUnit {@link org.junit.Rule} that detects the fact that RabbitMQ is available on localhost with + * the management plugin enabled. + * + * @author Gary Russell + * @since 1.2 + */ +public class RabbitAdminTestSupport extends AbstractExternalResourceTestSupport { + + public RabbitAdminTestSupport() { + super("RABBITADMIN"); + } + + @Override + protected void obtainResource() throws Exception { + resource = new RestTemplate(); + try { + resource.getForObject("http://localhost:15672/api/overview", Map.class); + } + catch (HttpClientErrorException e) { + if (e.getStatusCode() != HttpStatus.UNAUTHORIZED) { + throw e; + } + } + } + + @Override + protected void cleanupResource() throws Exception { + } + +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/test/java/org/springframework/xd/dirt/integration/rabbit/RabbitBusCleanerTests.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/test/java/org/springframework/xd/dirt/integration/rabbit/RabbitBusCleanerTests.java new file mode 100644 index 000000000..0416ac309 --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/test/java/org/springframework/xd/dirt/integration/rabbit/RabbitBusCleanerTests.java @@ -0,0 +1,216 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.xd.dirt.integration.rabbit; + + +import java.net.URI; +import java.util.List; +import java.util.Map; +import java.util.UUID; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.rabbitmq.client.Channel; +import com.rabbitmq.client.DefaultConsumer; +import org.junit.Rule; +import org.junit.Test; + +import org.springframework.amqp.core.BindingBuilder; +import org.springframework.amqp.core.FanoutExchange; +import org.springframework.amqp.core.Queue; +import org.springframework.amqp.rabbit.connection.CachingConnectionFactory; +import org.springframework.amqp.rabbit.core.ChannelCallback; +import org.springframework.amqp.rabbit.core.RabbitAdmin; +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.web.client.RestTemplate; +import org.springframework.web.util.UriComponentsBuilder; +import org.springframework.xd.dirt.integration.bus.BusUtils; +import org.springframework.xd.dirt.integration.bus.MessageBusSupport; +import org.springframework.xd.dirt.integration.bus.RabbitAdminException; +import org.springframework.xd.dirt.integration.bus.RabbitManagementUtils; + +import static org.hamcrest.Matchers.startsWith; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * @author Gary Russell + * @since 1.2 + */ +public class RabbitBusCleanerTests { + + private static final String XDBUS_PREFIX = "xdbus."; + + @Rule + public RabbitAdminTestSupport adminTest = new RabbitAdminTestSupport(); + + @Rule + public RabbitTestSupport test = new RabbitTestSupport(); + + @Test + public void testCleanStream() { + final RabbitBusCleaner cleaner = new RabbitBusCleaner(); + final RestTemplate template = RabbitManagementUtils.buildRestTemplate("http://localhost:15672", "guest", + "guest"); + final String stream1 = UUID.randomUUID().toString(); + String stream2 = stream1 + "-1"; + String firstQueue = null; + for (int i = 0; i < 5; i++) { + String queue1Name = MessageBusSupport.applyPrefix(XDBUS_PREFIX, + BusUtils.constructPipeName(stream1, i)); + String queue2Name = MessageBusSupport.applyPrefix(XDBUS_PREFIX, + BusUtils.constructPipeName(stream2, i)); + if (firstQueue == null) { + firstQueue = queue1Name; + } + URI uri = UriComponentsBuilder.fromUriString("http://localhost:15672/api/queues") + .pathSegment("{vhost}", "{queue}") + .buildAndExpand("/", queue1Name) + .encode().toUri(); + template.put(uri, new AmqpQueue(false, true)); + uri = UriComponentsBuilder.fromUriString("http://localhost:15672/api/queues") + .pathSegment("{vhost}", "{queue}") + .buildAndExpand("/", queue2Name) + .encode().toUri(); + template.put(uri, new AmqpQueue(false, true)); + uri = UriComponentsBuilder.fromUriString("http://localhost:15672/api/queues") + .pathSegment("{vhost}", "{queue}") + .buildAndExpand("/", MessageBusSupport.constructDLQName(queue1Name)).encode().toUri(); + template.put(uri, new AmqpQueue(false, true)); + } + CachingConnectionFactory connectionFactory = test.getResource(); + RabbitAdmin rabbitAdmin = new RabbitAdmin(connectionFactory); + final FanoutExchange fanout1 = new FanoutExchange( + MessageBusSupport.applyPrefix(XDBUS_PREFIX, MessageBusSupport.applyPubSub( + BusUtils.constructTapPrefix(stream1) + ".foo.bar"))); + rabbitAdmin.declareExchange(fanout1); + rabbitAdmin.declareBinding(BindingBuilder.bind(new Queue(firstQueue)).to(fanout1)); + final FanoutExchange fanout2 = new FanoutExchange( + MessageBusSupport.applyPrefix(XDBUS_PREFIX, MessageBusSupport.applyPubSub( + BusUtils.constructTapPrefix(stream2) + ".foo.bar"))); + rabbitAdmin.declareExchange(fanout2); + rabbitAdmin.declareBinding(BindingBuilder.bind(new Queue(firstQueue)).to(fanout2)); + new RabbitTemplate(connectionFactory).execute(new ChannelCallback() { + + @Override + public Void doInRabbit(Channel channel) throws Exception { + String queueName = MessageBusSupport.applyPrefix(XDBUS_PREFIX, + BusUtils.constructPipeName(stream1, 4)); + String consumerTag = channel.basicConsume(queueName, new DefaultConsumer(channel)); + try { + waitForConsumerStateNot(queueName, 0); + cleaner.clean(stream1, false); + fail("Expected exception"); + } + catch (RabbitAdminException e) { + assertEquals("Queue " + queueName + " is in use", e.getMessage()); + } + channel.basicCancel(consumerTag); + waitForConsumerStateNot(queueName, 1); + try { + cleaner.clean(stream1, false); + fail("Expected exception"); + } + catch (RabbitAdminException e) { + assertThat(e.getMessage(), startsWith("Cannot delete exchange " + + fanout1.getName() + "; it has bindings:")); + } + return null; + } + + private void waitForConsumerStateNot(String queueName, int state) throws InterruptedException { + int n = 0; + URI uri = UriComponentsBuilder.fromUriString("http://localhost:15672/api/queues").pathSegment( + "{vhost}", "{queue}") + .buildAndExpand("/", queueName).encode().toUri(); + while (n++ < 100) { + @SuppressWarnings("unchecked") + Map queueInfo = template.getForObject(uri, Map.class); + if (!queueInfo.get("consumers").equals(Integer.valueOf(state))) { + break; + } + Thread.sleep(100); + } + assertTrue("Consumer state remained at " + state + " after 10 seconds", n < 100); + } + + }); + rabbitAdmin.deleteExchange(fanout1.getName()); // easier than deleting the binding + rabbitAdmin.declareExchange(fanout1); + connectionFactory.destroy(); + Map> cleanedMap = cleaner.clean(stream1, false); + assertEquals(2, cleanedMap.size()); + List cleanedQueues = cleanedMap.get("queues"); + // should *not* clean stream2 + assertEquals(10, cleanedQueues.size()); + for (int i = 0; i < 5; i++) { + assertEquals(XDBUS_PREFIX + stream1 + "." + i, cleanedQueues.get(i * 2)); + assertEquals(XDBUS_PREFIX + stream1 + "." + i + ".dlq", cleanedQueues.get(i * 2 + 1)); + } + List cleanedExchanges = cleanedMap.get("exchanges"); + assertEquals(1, cleanedExchanges.size()); + assertEquals(fanout1.getName(), cleanedExchanges.get(0)); + + // wild card *should* clean stream2 + cleanedMap = cleaner.clean(stream1 + "*", false); + assertEquals(2, cleanedMap.size()); + cleanedQueues = cleanedMap.get("queues"); + assertEquals(5, cleanedQueues.size()); + for (int i = 0; i < 5; i++) { + assertEquals(XDBUS_PREFIX + stream2 + "." + i, cleanedQueues.get(i)); + } + cleanedExchanges = cleanedMap.get("exchanges"); + assertEquals(1, cleanedExchanges.size()); + assertEquals(fanout2.getName(), cleanedExchanges.get(0)); + } + + public static class AmqpQueue { + + private boolean autoDelete; + + private boolean durable; + + public AmqpQueue(boolean autoDelete, boolean durable) { + this.autoDelete = autoDelete; + this.durable = durable; + } + + + @JsonProperty("auto_delete") + protected boolean isAutoDelete() { + return autoDelete; + } + + + protected void setAutoDelete(boolean autoDelete) { + this.autoDelete = autoDelete; + } + + + protected boolean isDurable() { + return durable; + } + + + protected void setDurable(boolean durable) { + this.durable = durable; + } + + } + +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/test/java/org/springframework/xd/dirt/integration/rabbit/RabbitMessageBusTests.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/test/java/org/springframework/xd/dirt/integration/rabbit/RabbitMessageBusTests.java new file mode 100644 index 000000000..c3860a8ff --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/test/java/org/springframework/xd/dirt/integration/rabbit/RabbitMessageBusTests.java @@ -0,0 +1,715 @@ +/* + * Copyright 2013-2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.xd.dirt.integration.rabbit; + +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.startsWith; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.zip.Deflater; + +import org.aopalliance.aop.Advice; +import org.apache.commons.logging.Log; +import org.junit.Rule; +import org.junit.Test; +import org.mockito.ArgumentCaptor; + +import org.springframework.amqp.core.AcknowledgeMode; +import org.springframework.amqp.core.MessageDeliveryMode; +import org.springframework.amqp.core.Queue; +import org.springframework.amqp.rabbit.core.RabbitAdmin; +import org.springframework.amqp.rabbit.core.RabbitTemplate; +import org.springframework.amqp.rabbit.listener.SimpleMessageListenerContainer; +import org.springframework.amqp.support.AmqpHeaders; +import org.springframework.amqp.support.postprocessor.DelegatingDecompressingPostProcessor; +import org.springframework.amqp.utils.test.TestUtils; +import org.springframework.beans.DirectFieldAccessor; +import org.springframework.expression.spel.standard.SpelExpression; +import org.springframework.integration.channel.DirectChannel; +import org.springframework.integration.channel.QueueChannel; +import org.springframework.integration.endpoint.AbstractEndpoint; +import org.springframework.integration.support.MessageBuilder; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageHandler; +import org.springframework.messaging.MessageHeaders; +import org.springframework.messaging.MessagingException; +import org.springframework.messaging.support.GenericMessage; +import org.springframework.xd.dirt.integration.bus.Binding; +import org.springframework.xd.dirt.integration.bus.BusProperties; +import org.springframework.xd.dirt.integration.bus.MessageBus; +import org.springframework.xd.dirt.integration.bus.PartitionCapableBusTests; +import org.springframework.xd.dirt.integration.bus.Spy; + +/** + * @author Mark Fisher + * @author Gary Russell + */ +public class RabbitMessageBusTests extends PartitionCapableBusTests { + + @Rule + public RabbitTestSupport rabbitAvailableRule = new RabbitTestSupport(); + + @Override + protected MessageBus getMessageBus() { + if (testMessageBus == null) { + testMessageBus = new RabbitTestMessageBus(rabbitAvailableRule.getResource(), getCodec()); + } + return testMessageBus; + } + + @Override + protected boolean usesExplicitRouting() { + return true; + } + + @Test + public void testSendAndReceiveBad() throws Exception { + MessageBus messageBus = getMessageBus(); + DirectChannel moduleOutputChannel = new DirectChannel(); + DirectChannel moduleInputChannel = new DirectChannel(); + messageBus.bindProducer("bad.0", moduleOutputChannel, null); + messageBus.bindConsumer("bad.0", moduleInputChannel, null); + Message message = MessageBuilder.withPayload("bad").setHeader(MessageHeaders.CONTENT_TYPE, "foo/bar").build(); + final CountDownLatch latch = new CountDownLatch(3); + moduleInputChannel.subscribe(new MessageHandler() { + + @Override + public void handleMessage(Message message) throws MessagingException { + latch.countDown(); + throw new RuntimeException("bad"); + } + }); + moduleOutputChannel.send(message); + assertTrue(latch.await(10, TimeUnit.SECONDS)); + messageBus.unbindConsumers("bad.0"); + messageBus.unbindProducers("bad.0"); + } + + @Test + public void testConsumerProperties() throws Exception { + MessageBus bus = getMessageBus(); + Properties properties = new Properties(); + properties.put("transacted", "true"); // test transacted with defaults; not allowed with ackmode NONE + bus.bindConsumer("props.0", new DirectChannel(), properties); + @SuppressWarnings("unchecked") + List bindings = TestUtils.getPropertyValue(bus, "messageBus.bindings", List.class); + assertEquals(1, bindings.size()); + AbstractEndpoint endpoint = bindings.get(0).getEndpoint(); + SimpleMessageListenerContainer container = TestUtils.getPropertyValue(endpoint, "messageListenerContainer", + SimpleMessageListenerContainer.class); + assertEquals(AcknowledgeMode.AUTO, container.getAcknowledgeMode()); + assertEquals("xdbus.props.0", container.getQueueNames()[0]); + assertTrue(TestUtils.getPropertyValue(container, "transactional", Boolean.class)); + assertEquals(1, TestUtils.getPropertyValue(container, "concurrentConsumers")); + assertNull(TestUtils.getPropertyValue(container, "maxConcurrentConsumers")); + assertTrue(TestUtils.getPropertyValue(container, "defaultRequeueRejected", Boolean.class)); + assertEquals(1, TestUtils.getPropertyValue(container, "prefetchCount")); + assertEquals(1, TestUtils.getPropertyValue(container, "txSize")); + Advice retry = TestUtils.getPropertyValue(container, "adviceChain", Advice[].class)[0]; + assertEquals(3, TestUtils.getPropertyValue(retry, "retryOperations.retryPolicy.maxAttempts")); + assertEquals(1000L, TestUtils.getPropertyValue(retry, "retryOperations.backOffPolicy.initialInterval")); + assertEquals(10000L, TestUtils.getPropertyValue(retry, "retryOperations.backOffPolicy.maxInterval")); + assertEquals(2.0, TestUtils.getPropertyValue(retry, "retryOperations.backOffPolicy.multiplier")); + bus.unbindConsumers("props.0"); + assertEquals(0, bindings.size()); + + properties = new Properties(); + properties.put("ackMode", "NONE"); + properties.put("backOffInitialInterval", "2000"); + properties.put("backOffMaxInterval", "20000"); + properties.put("backOffMultiplier", "5.0"); + properties.put("concurrency", "2"); + properties.put("maxAttempts", "23"); + properties.put("maxConcurrency", "3"); + properties.put("prefix", "foo."); + properties.put("prefetch", "20"); + properties.put("requestHeaderPatterns", "foo"); + properties.put("requeue", "false"); + properties.put("txSize", "10"); + properties.put("partitionIndex", 0); + bus.bindConsumer("props.0", new DirectChannel(), properties); + + @SuppressWarnings("unchecked") + List bindingsNow = TestUtils.getPropertyValue(bus, "messageBus.bindings", List.class); + assertEquals(1, bindingsNow.size()); + endpoint = bindingsNow.get(0).getEndpoint(); + container = verifyContainer(endpoint); + + assertEquals("foo.props.0", container.getQueueNames()[0]); + + try { + bus.bindPubSubConsumer("dummy", null, properties); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertThat(e.getMessage(), allOf( + containsString("RabbitMessageBus does not support consumer properties: "), + containsString("partitionIndex"), + containsString("concurrency"), + containsString(" for dummy."))); + } + try { + bus.bindConsumer("queue:dummy", null, properties); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("RabbitMessageBus does not support consumer property: partitionIndex for queue:dummy.", + e.getMessage()); + } + + bus.unbindConsumers("props.0"); + assertEquals(0, bindingsNow.size()); + } + + @Test + public void testProducerProperties() throws Exception { + MessageBus bus = getMessageBus(); + bus.bindProducer("props.0", new DirectChannel(), null); + @SuppressWarnings("unchecked") + List bindings = TestUtils.getPropertyValue(bus, "messageBus.bindings", List.class); + assertEquals(1, bindings.size()); + AbstractEndpoint endpoint = bindings.get(0).getEndpoint(); + assertEquals("xdbus.props.0", TestUtils.getPropertyValue(endpoint, "handler.delegate.routingKey")); + MessageDeliveryMode mode = TestUtils.getPropertyValue(endpoint, "handler.delegate.defaultDeliveryMode", + MessageDeliveryMode.class); + assertEquals(MessageDeliveryMode.PERSISTENT, mode); + List requestHeaders = TestUtils.getPropertyValue(endpoint, + "handler.delegate.headerMapper.requestHeaderMatcher.strategies", List.class); + assertEquals(2, requestHeaders.size()); + bus.unbindProducers("props.0"); + assertEquals(0, bindings.size()); + + Properties properties = new Properties(); + properties.put("prefix", "foo."); + properties.put("deliveryMode", "NON_PERSISTENT"); + properties.put("requestHeaderPatterns", "foo"); + properties.put("partitionKeyExpression", "'foo'"); + properties.put("partitionKeyExtractorClass", "foo"); + properties.put("partitionSelectorExpression", "0"); + properties.put("partitionSelectorClass", "foo"); + properties.put(BusProperties.NEXT_MODULE_COUNT, "1"); + + bus.bindProducer("props.0", new DirectChannel(), properties); + assertEquals(1, bindings.size()); + endpoint = bindings.get(0).getEndpoint(); + assertEquals( + "'foo.props.0-' + headers['partition']", + TestUtils.getPropertyValue(endpoint, "handler.delegate.routingKeyExpression", SpelExpression.class).getExpressionString()); + mode = TestUtils.getPropertyValue(endpoint, "handler.delegate.defaultDeliveryMode", + MessageDeliveryMode.class); + assertEquals(MessageDeliveryMode.NON_PERSISTENT, mode); + verifyFooRequestProducer(endpoint); + + try { + bus.bindPubSubProducer("dummy", new DirectChannel(), properties); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertThat(e.getMessage(), allOf( + containsString("RabbitMessageBus does not support producer properties: "), + containsString("partitionSelectorExpression"), + containsString("partitionKeyExtractorClass"), + containsString("partitionKeyExpression"), + containsString("partitionSelectorClass"))); + assertThat(e.getMessage(), containsString("for dummy.")); + } + try { + bus.bindProducer("queue:dummy", new DirectChannel(), properties); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertThat(e.getMessage(), allOf( + containsString("RabbitMessageBus does not support producer properties: "), + containsString("partitionSelectorExpression"), + containsString("partitionKeyExtractorClass"), + containsString("partitionKeyExpression"), + containsString("partitionSelectorClass"))); + assertThat(e.getMessage(), containsString("for queue:dummy.")); + } + + bus.unbindProducers("props.0"); + assertEquals(0, bindings.size()); + } + + @Test + public void testRequestReplyRequestorProperties() throws Exception { + MessageBus bus = getMessageBus(); + Properties properties = new Properties(); + properties.put("prefix", "foo."); + properties.put("deliveryMode", "NON_PERSISTENT"); + + properties.put("requestHeaderPatterns", "foo"); + properties.put("replyHeaderPatterns", "bar"); + + properties.put("ackMode", "NONE"); + properties.put("backOffInitialInterval", "2000"); + properties.put("backOffMaxInterval", "20000"); + properties.put("backOffMultiplier", "5.0"); + properties.put("concurrency", "2"); + properties.put("maxAttempts", "23"); + properties.put("maxConcurrency", "3"); + properties.put("prefix", "foo."); + properties.put("prefetch", "20"); + properties.put("requeue", "false"); + properties.put("txSize", "10"); + + bus.bindRequestor("props.0", new DirectChannel(), new DirectChannel(), properties); + @SuppressWarnings("unchecked") + List bindings = TestUtils.getPropertyValue(bus, "messageBus.bindings", List.class); + + assertEquals(2, bindings.size()); + AbstractEndpoint endpoint = bindings.get(0).getEndpoint(); // producer + assertEquals("foo.props.0.requests", + TestUtils.getPropertyValue(endpoint, "handler.delegate.routingKey")); + MessageDeliveryMode mode = TestUtils.getPropertyValue(endpoint, "handler.delegate.defaultDeliveryMode", + MessageDeliveryMode.class); + assertEquals(MessageDeliveryMode.NON_PERSISTENT, mode); + verifyFooRequestBarReplyProducer(endpoint); + + endpoint = bindings.get(1).getEndpoint(); // consumer + + verifyContainer(endpoint); + + verifyBarReplyConsumer(endpoint); + + properties.put("partitionKeyExpression", "'foo'"); + properties.put("partitionKeyExtractorClass", "foo"); + properties.put("partitionSelectorExpression", "0"); + properties.put("partitionSelectorClass", "foo"); + properties.put(BusProperties.NEXT_MODULE_COUNT, "1"); + properties.put("partitionIndex", "0"); + try { + bus.bindRequestor("dummy", null, null, properties); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertThat(e.getMessage(), allOf( + containsString("RabbitMessageBus does not support producer properties: "), + containsString("partitionSelectorExpression"), + containsString("partitionKeyExtractorClass"), + containsString("partitionKeyExpression"), + containsString("partitionSelectorClass"))); + assertThat(e.getMessage(), allOf(containsString("partitionIndex"), containsString("for dummy."))); + } + + bus.unbindConsumers("props.0"); + bus.unbindProducers("props.0"); + assertEquals(0, bindings.size()); + } + + @Test + public void testRequestReplyReplierProperties() throws Exception { + MessageBus bus = getMessageBus(); + Properties properties = new Properties(); + properties.put("prefix", "foo."); + properties.put("deliveryMode", "NON_PERSISTENT"); + + properties.put("requestHeaderPatterns", "foo"); + properties.put("replyHeaderPatterns", "bar"); + + properties.put("ackMode", "NONE"); + properties.put("backOffInitialInterval", "2000"); + properties.put("backOffMaxInterval", "20000"); + properties.put("backOffMultiplier", "5.0"); + properties.put("concurrency", "2"); + properties.put("maxAttempts", "23"); + properties.put("maxConcurrency", "3"); + properties.put("prefix", "foo."); + properties.put("prefetch", "20"); + properties.put("requeue", "false"); + properties.put("txSize", "10"); + + bus.bindReplier("props.0", new DirectChannel(), new DirectChannel(), properties); + @SuppressWarnings("unchecked") + List bindings = TestUtils.getPropertyValue(bus, "messageBus.bindings", List.class); + + assertEquals(2, bindings.size()); + AbstractEndpoint endpoint = bindings.get(1).getEndpoint(); // producer + assertEquals( + "headers['amqp_replyTo']", + TestUtils.getPropertyValue(endpoint, "handler.delegate.routingKeyExpression", SpelExpression.class).getExpressionString()); + MessageDeliveryMode mode = TestUtils.getPropertyValue(endpoint, "handler.delegate.defaultDeliveryMode", + MessageDeliveryMode.class); + assertEquals(MessageDeliveryMode.NON_PERSISTENT, mode); + + verifyFooRequestBarReplyProducer(endpoint); + + endpoint = bindings.get(0).getEndpoint(); // consumer + + verifyContainer(endpoint); + + verifyBarReplyConsumer(endpoint); + + properties.put("partitionKeyExpression", "'foo'"); + properties.put("partitionKeyExtractorClass", "foo"); + properties.put("partitionSelectorExpression", "0"); + properties.put("partitionSelectorClass", "foo"); + properties.put(BusProperties.NEXT_MODULE_COUNT, "1"); + properties.put("partitionIndex", "0"); + try { + bus.bindReplier("dummy", null, null, properties); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertThat(e.getMessage(), allOf( + containsString("RabbitMessageBus does not support consumer properties: "), + containsString("partitionSelectorExpression"), + containsString("partitionKeyExtractorClass"), + containsString("partitionKeyExpression"), + containsString("partitionSelectorClass"))); + assertThat(e.getMessage(), allOf(containsString("partitionIndex"), containsString("for dummy."))); + } + + bus.unbindConsumers("props.0"); + bus.unbindProducers("props.0"); + assertEquals(0, bindings.size()); + } + + @Test + public void testDurablePubSubWithAutoBindDLQ() throws Exception { + RabbitAdmin admin = new RabbitAdmin(this.rabbitAvailableRule.getResource()); + + MessageBus bus = getMessageBus(); + Properties properties = new Properties(); + properties.put("prefix", "xdbustest."); + properties.put("autoBindDLQ", "true"); + properties.put("durableSubscription", "true"); + properties.put("maxAttempts", "1"); // disable retry + properties.put("requeue", "false"); + DirectChannel moduleInputChannel = new DirectChannel(); + moduleInputChannel.setBeanName("durableTest"); + moduleInputChannel.subscribe(new MessageHandler() { + + @Override + public void handleMessage(Message message) throws MessagingException { + throw new RuntimeException("foo"); + } + + }); + bus.bindPubSubConsumer("teststream.tap:stream:durabletest.0", moduleInputChannel, properties); + + RabbitTemplate template = new RabbitTemplate(this.rabbitAvailableRule.getResource()); + template.convertAndSend("xdbustest.topic.tap:stream:durabletest.0", "", "foo"); + + int n = 0; + while (n++ < 100) { + Object deadLetter = template.receiveAndConvert("xdbustest.teststream.tap:stream:durabletest.0.dlq"); + if (deadLetter != null) { + assertEquals("foo", deadLetter); + break; + } + Thread.sleep(100); + } + assertTrue(n < 100); + + bus.unbindConsumer("teststream.tap:stream:durabletest.0", moduleInputChannel); + assertNotNull(admin.getQueueProperties("xdbustest.teststream.tap:stream:durabletest.0.dlq")); + admin.deleteQueue("xdbustest.teststream.tap:stream:durabletest.0.dlq"); + admin.deleteQueue("xdbustest.teststream.tap:stream:durabletest.0"); + admin.deleteExchange("xdbustest.topic.tap:stream:durabletest.0"); + admin.deleteExchange("xdbustest.DLX"); + } + + @Test + public void testNonDurablePubSubWithAutoBindDLQ() throws Exception { + RabbitAdmin admin = new RabbitAdmin(this.rabbitAvailableRule.getResource()); + + MessageBus bus = getMessageBus(); + Properties properties = new Properties(); + properties.put("prefix", "xdbustest."); + properties.put("autoBindDLQ", "true"); + properties.put("durableSubscription", "false"); + properties.put("maxAttempts", "1"); // disable retry + properties.put("requeue", "false"); + DirectChannel moduleInputChannel = new DirectChannel(); + moduleInputChannel.setBeanName("nondurabletest"); + moduleInputChannel.subscribe(new MessageHandler() { + + @Override + public void handleMessage(Message message) throws MessagingException { + throw new RuntimeException("foo"); + } + + }); + bus.bindPubSubConsumer("teststream.tap:stream:nondurabletest.0", moduleInputChannel, properties); + + bus.unbindConsumer("teststream.tap:stream:nondurabletest.0", moduleInputChannel); + assertNull(admin.getQueueProperties("xdbustest.teststream.tap:stream:nondurabletest.0.dlq")); + admin.deleteQueue("xdbustest.teststream.tap:stream:nondurabletest.0"); + admin.deleteExchange("xdbustest.topic.tap:stream:nondurabletest.0"); + } + + @Test + public void testAutoBindDLQ() throws Exception { + RabbitAdmin admin = new RabbitAdmin(this.rabbitAvailableRule.getResource()); + + MessageBus bus = getMessageBus(); + Properties properties = new Properties(); + properties.put("prefix", "xdbustest."); + properties.put("autoBindDLQ", "true"); + properties.put("maxAttempts", "1"); // disable retry + properties.put("requeue", "false"); + DirectChannel moduleInputChannel = new DirectChannel(); + moduleInputChannel.setBeanName("dlqTest"); + moduleInputChannel.subscribe(new MessageHandler() { + + @Override + public void handleMessage(Message message) throws MessagingException { + throw new RuntimeException("foo"); + } + + }); + bus.bindConsumer("dlqtest", moduleInputChannel, properties); + + RabbitTemplate template = new RabbitTemplate(this.rabbitAvailableRule.getResource()); + template.convertAndSend("", "xdbustest.dlqtest", "foo"); + + int n = 0; + while (n++ < 100) { + Object deadLetter = template.receiveAndConvert("xdbustest.dlqtest.dlq"); + if (deadLetter != null) { + assertEquals("foo", deadLetter); + break; + } + Thread.sleep(100); + } + assertTrue(n < 100); + + bus.unbindConsumer("dlqtest", moduleInputChannel); + admin.deleteQueue("xdbustest.dlqtest.dlq"); + admin.deleteQueue("xdbustest.dlqtest"); + admin.deleteExchange("xdbustest.DLX"); + } + + @Test + public void testAutoBindDLQwithRepublish() throws Exception { + // pre-declare the queue with dead-lettering, users can also use a policy + RabbitAdmin admin = new RabbitAdmin(this.rabbitAvailableRule.getResource()); + Map args = new HashMap(); + args.put("x-dead-letter-exchange", "xdbustest.DLX"); + Queue queue = new Queue("xdbustest.dlqpubtest", true, false, false, args); + admin.declareQueue(queue); + + MessageBus bus = getMessageBus(); + Properties properties = new Properties(); + properties.put("prefix", "xdbustest."); + properties.put("autoBindDLQ", "true"); + properties.put("republishToDLQ", "true"); + properties.put("maxAttempts", "1"); // disable retry + properties.put("requeue", "false"); + DirectChannel moduleInputChannel = new DirectChannel(); + moduleInputChannel.setBeanName("dlqPubTest"); + moduleInputChannel.subscribe(new MessageHandler() { + + @Override + public void handleMessage(Message message) throws MessagingException { + throw new RuntimeException("foo"); + } + + }); + bus.bindConsumer("dlqpubtest", moduleInputChannel, properties); + + RabbitTemplate template = new RabbitTemplate(this.rabbitAvailableRule.getResource()); + template.convertAndSend("", "xdbustest.dlqpubtest", "foo"); + + int n = 0; + while (n++ < 100) { + org.springframework.amqp.core.Message deadLetter = template.receive("xdbustest.dlqpubtest.dlq"); + if (deadLetter != null) { + assertEquals("foo", new String(deadLetter.getBody())); + assertNotNull(deadLetter.getMessageProperties().getHeaders().get("x-exception-stacktrace")); + break; + } + Thread.sleep(100); + } + assertTrue(n < 100); + + bus.unbindConsumer("dlqpubtest", moduleInputChannel); + admin.deleteQueue("xdbustest.dlqpubtest.dlq"); + admin.deleteQueue("xdbustest.dlqpubtest"); + admin.deleteExchange("xdbustest.DLX"); + } + + @SuppressWarnings("unchecked") + @Test + public void testBatchingAndCompression() throws Exception { + RabbitTemplate template = new RabbitTemplate(this.rabbitAvailableRule.getResource()); + MessageBus bus = getMessageBus(); + Properties properties = new Properties(); + properties.put("deliveryMode", "NON_PERSISTENT"); + properties.put("batchingEnabled", "true"); + properties.put("batchSize", "2"); + properties.put("batchBufferLimit", "100000"); + properties.put("batchTimeout", "30000"); + properties.put("compress", "true"); + + DirectChannel output = new DirectChannel(); + output.setBeanName("batchingProducer"); + bus.bindProducer("batching.0", output, properties); + + while (template.receive("xdbus.batching.0") != null) { + } + + Log logger = spy(TestUtils.getPropertyValue(bus, "messageBus.compressingPostProcessor.logger", Log.class)); + new DirectFieldAccessor(TestUtils.getPropertyValue(bus, "messageBus.compressingPostProcessor")) + .setPropertyValue("logger", logger); + when(logger.isTraceEnabled()).thenReturn(true); + + assertEquals(Deflater.BEST_SPEED, TestUtils.getPropertyValue(bus, "messageBus.compressingPostProcessor.level")); + + output.send(new GenericMessage<>("foo".getBytes())); + output.send(new GenericMessage<>("bar".getBytes())); + + Object out = spyOn("batching.0").receive(false); + assertThat(out, instanceOf(byte[].class)); + assertEquals("\u0000\u0000\u0000\u0003foo\u0000\u0000\u0000\u0003bar", new String((byte[]) out)); + + ArgumentCaptor captor = ArgumentCaptor.forClass(Object.class); + verify(logger).trace(captor.capture()); + assertThat(captor.getValue().toString(), containsString("Compressed 14 to ")); + + QueueChannel input = new QueueChannel(); + input.setBeanName("batchingConsumer"); + bus.bindConsumer("batching.0", input, null); + + output.send(new GenericMessage<>("foo".getBytes())); + output.send(new GenericMessage<>("bar".getBytes())); + + Message in = (Message) input.receive(10000); + assertNotNull(in); + assertEquals("foo", new String(in.getPayload())); + in = (Message) input.receive(10000); + assertNotNull(in); + assertEquals("bar", new String(in.getPayload())); + assertNull(in.getHeaders().get(AmqpHeaders.DELIVERY_MODE)); + + bus.unbindProducers("batching.0"); + bus.unbindConsumers("batching.0"); + } + + private SimpleMessageListenerContainer verifyContainer(AbstractEndpoint endpoint) { + SimpleMessageListenerContainer container; + Advice retry; + container = TestUtils.getPropertyValue(endpoint, "messageListenerContainer", + SimpleMessageListenerContainer.class); + assertEquals(AcknowledgeMode.NONE, container.getAcknowledgeMode()); + assertThat(container.getQueueNames()[0], startsWith("foo.props.0")); + assertFalse(TestUtils.getPropertyValue(container, "transactional", Boolean.class)); + assertEquals(2, TestUtils.getPropertyValue(container, "concurrentConsumers")); + assertEquals(3, TestUtils.getPropertyValue(container, "maxConcurrentConsumers")); + assertFalse(TestUtils.getPropertyValue(container, "defaultRequeueRejected", Boolean.class)); + assertEquals(20, TestUtils.getPropertyValue(container, "prefetchCount")); + assertEquals(10, TestUtils.getPropertyValue(container, "txSize")); + retry = TestUtils.getPropertyValue(container, "adviceChain", Advice[].class)[0]; + assertEquals(23, TestUtils.getPropertyValue(retry, "retryOperations.retryPolicy.maxAttempts")); + assertEquals(2000L, TestUtils.getPropertyValue(retry, "retryOperations.backOffPolicy.initialInterval")); + assertEquals(20000L, TestUtils.getPropertyValue(retry, "retryOperations.backOffPolicy.maxInterval")); + assertEquals(5.0, TestUtils.getPropertyValue(retry, "retryOperations.backOffPolicy.multiplier")); + + List requestMatchers = TestUtils.getPropertyValue(endpoint, + "headerMapper.requestHeaderMatcher.strategies", + List.class); + assertEquals(1, requestMatchers.size()); + assertEquals("foo", + TestUtils.getPropertyValue(requestMatchers.get(0), "patterns", Collection.class).iterator().next()); + + return container; + } + + private void verifyBarReplyConsumer(AbstractEndpoint endpoint) { + List replyMatchers; + replyMatchers = TestUtils.getPropertyValue(endpoint, + "headerMapper.replyHeaderMatcher.strategies", + List.class); + assertEquals(1, replyMatchers.size()); + assertEquals("bar", + TestUtils.getPropertyValue(replyMatchers.get(0), "patterns", Collection.class).iterator().next()); + } + + private void verifyFooRequestBarReplyProducer(AbstractEndpoint endpoint) { + verifyFooRequestProducer(endpoint); + List replyMatchers = TestUtils.getPropertyValue(endpoint, + "handler.delegate.headerMapper.replyHeaderMatcher.strategies", + List.class); + assertEquals(1, replyMatchers.size()); + assertEquals("bar", + TestUtils.getPropertyValue(replyMatchers.get(0), "patterns", Collection.class).iterator().next()); + } + + private void verifyFooRequestProducer(AbstractEndpoint endpoint) { + List requestMatchers = TestUtils.getPropertyValue(endpoint, + "handler.delegate.headerMapper.requestHeaderMatcher.strategies", + List.class); + assertEquals(1, requestMatchers.size()); + assertEquals("foo", + TestUtils.getPropertyValue(requestMatchers.get(0), "patterns", Collection.class).iterator().next()); + } + + @Override + protected String getEndpointRouting(AbstractEndpoint endpoint) { + return TestUtils.getPropertyValue(endpoint, "handler.delegate.routingKeyExpression", SpelExpression.class).getExpressionString(); + } + + @Override + protected String getPubSubEndpointRouting(AbstractEndpoint endpoint) { + return TestUtils.getPropertyValue(endpoint, "handler.delegate.exchangeNameExpression", SpelExpression.class).getExpressionString(); + } + + @Override + public Spy spyOn(final String queue) { + final RabbitTemplate template = new RabbitTemplate(this.rabbitAvailableRule.getResource()); + template.setAfterReceivePostProcessors(new DelegatingDecompressingPostProcessor()); + return new Spy() { + + @Override + public Object receive(boolean expectNull) throws Exception { + if (expectNull) { + Thread.sleep(50); + return template.receiveAndConvert("xdbus." + queue); + } + Object bar = null; + int n = 0; + while (n++ < 100 && bar == null) { + bar = template.receiveAndConvert("xdbus." + queue); + Thread.sleep(100); + } + assertTrue("Message did not arrive in RabbitMQ", n < 100); + return bar; + } + + }; + } + +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/test/java/org/springframework/xd/dirt/integration/rabbit/RabbitTestMessageBus.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/test/java/org/springframework/xd/dirt/integration/rabbit/RabbitTestMessageBus.java new file mode 100644 index 000000000..59b45e103 --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/test/java/org/springframework/xd/dirt/integration/rabbit/RabbitTestMessageBus.java @@ -0,0 +1,77 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.xd.dirt.integration.rabbit; + +import org.springframework.amqp.rabbit.connection.ConnectionFactory; +import org.springframework.amqp.rabbit.core.RabbitAdmin; +import org.springframework.context.support.GenericApplicationContext; +import org.springframework.integration.context.IntegrationContextUtils; +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler; +import org.springframework.xd.dirt.integration.bus.AbstractTestMessageBus; +import org.springframework.xd.dirt.integration.bus.serializer.MultiTypeCodec; + + +/** + * Test support class for {@link org.springframework.xd.dirt.integration.rabbit.RabbitMessageBus}. + * + * @author Ilayaperumal Gopinathan + * @author Gary Russell + */ +public class RabbitTestMessageBus extends AbstractTestMessageBus { + + private final RabbitAdmin rabbitAdmin; + + public RabbitTestMessageBus(ConnectionFactory connectionFactory) { + this.rabbitAdmin = new RabbitAdmin(connectionFactory); + } + + public RabbitTestMessageBus(ConnectionFactory connectionFactory, MultiTypeCodec codec) { + RabbitMessageBus messageBus = new RabbitMessageBus(connectionFactory, codec); + GenericApplicationContext context = new GenericApplicationContext(); + ThreadPoolTaskScheduler scheduler = new ThreadPoolTaskScheduler(); + scheduler.setPoolSize(1); + scheduler.afterPropertiesSet(); + context.getBeanFactory().registerSingleton(IntegrationContextUtils.TASK_SCHEDULER_BEAN_NAME, scheduler); + context.refresh(); + messageBus.setApplicationContext(context); + this.setMessageBus(messageBus); + this.rabbitAdmin = new RabbitAdmin(connectionFactory); + } + + @Override + public void cleanup() { + if (!queues.isEmpty()) { + for (String queue : queues) { + rabbitAdmin.deleteQueue("xdbus." + queue); + // delete any partitioned queues + for (int i = 0; i < 10; i++) { + rabbitAdmin.deleteQueue("xdbus." + queue + "-" + i); + } + rabbitAdmin.deleteQueue("foo." + queue); + // delete any partitioned queues + for (int i = 0; i < 10; i++) { + rabbitAdmin.deleteQueue("foo." + queue + "-" + i); + } + } + } + if (!topics.isEmpty()) { + for (String exchange : topics) { + rabbitAdmin.deleteExchange("xdbus." + exchange); + } + } + } +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/test/java/org/springframework/xd/dirt/integration/rabbit/RabbitTestSupport.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/test/java/org/springframework/xd/dirt/integration/rabbit/RabbitTestSupport.java new file mode 100644 index 000000000..200cdcefc --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/test/java/org/springframework/xd/dirt/integration/rabbit/RabbitTestSupport.java @@ -0,0 +1,61 @@ +/* + * Copyright 2013-2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.xd.dirt.integration.rabbit; + + +import org.springframework.amqp.rabbit.connection.CachingConnectionFactory; +import org.springframework.xd.test.AbstractExternalResourceTestSupport; + +import javax.net.SocketFactory; +import java.net.Socket; + +/** + * JUnit {@link org.junit.Rule} that detects the fact that RabbitMQ is available on localhost. + * + * @author Mark Fisher + * @author Gary Russell + * @author Eric Bottard + */ +public class RabbitTestSupport extends AbstractExternalResourceTestSupport { + + private final boolean management; + + public RabbitTestSupport() { + this(false); + } + + public RabbitTestSupport(boolean management) { + super("RABBIT"); + this.management = management; + } + + @Override + protected void obtainResource() throws Exception { + resource = new CachingConnectionFactory("localhost"); + resource.createConnection().close(); + if (management) { + Socket socket = SocketFactory.getDefault().createSocket("localhost", 15672); + socket.close(); + } + } + + @Override + protected void cleanupResource() throws Exception { + resource.destroy(); + } + +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/test/resources/log4j.properties b/spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/test/resources/log4j.properties new file mode 100644 index 000000000..c303f7f8c --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-rabbit/src/test/resources/log4j.properties @@ -0,0 +1,8 @@ +log4j.rootCategory=WARN, stdout + +# standard logging including calling site +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d{ABSOLUTE} %5p %40.40c:%4L - %m%n + +log4j.category.org.springframework.xd.dirt.integration.rabbit=DEBUG diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-redis/pom.xml b/spring-cloud-streams-bindings/spring-cloud-streams-binding-redis/pom.xml new file mode 100644 index 000000000..5e4c5ad33 --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-redis/pom.xml @@ -0,0 +1,58 @@ + + + 4.0.0 + + spring-cloud-streams-binding-redis + jar + spring-cloud-streams-binding-redis + Redis binding implementation + + + org.springframework.cloud + spring-cloud-streams-bindings-parent + 1.0.0.BUILD-SNAPSHOT + + + + UTF-8 + + + + + org.springframework.cloud + spring-cloud-streams-binding-spi + + + org.springframework.cloud + spring-cloud-streams-binding-test + + + org.springframework.boot + spring-boot-starter-redis + + + org.springframework.integration + spring-integration-redis + ${spring-integration.version} + + + joda-time + joda-time + 2.5 + test + + + org.springframework.xd + spring-xd-tuple + ${spring-xd.version} + + + org.springframework.xd + spring-xd-codec + + + test + + + diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-redis/src/main/java/org/springframework/xd/dirt/integration/redis/RedisMessageBus.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-redis/src/main/java/org/springframework/xd/dirt/integration/redis/RedisMessageBus.java new file mode 100644 index 000000000..f093a9c2b --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-redis/src/main/java/org/springframework/xd/dirt/integration/redis/RedisMessageBus.java @@ -0,0 +1,532 @@ +/* + * Copyright 2014-2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.xd.dirt.integration.redis; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Properties; +import java.util.Set; + +import org.springframework.beans.factory.DisposableBean; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.expression.Expression; +import org.springframework.expression.spel.standard.SpelExpressionParser; +import org.springframework.integration.channel.DirectChannel; +import org.springframework.integration.endpoint.EventDrivenConsumer; +import org.springframework.integration.endpoint.MessageProducerSupport; +import org.springframework.integration.handler.AbstractMessageHandler; +import org.springframework.integration.handler.AbstractReplyProducingMessageHandler; +import org.springframework.integration.redis.inbound.RedisInboundChannelAdapter; +import org.springframework.integration.redis.inbound.RedisQueueMessageDrivenEndpoint; +import org.springframework.integration.redis.outbound.RedisPublishingMessageHandler; +import org.springframework.integration.redis.outbound.RedisQueueOutboundChannelAdapter; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageChannel; +import org.springframework.messaging.MessageHandler; +import org.springframework.messaging.SubscribableChannel; +import org.springframework.messaging.support.MessageBuilder; +import org.springframework.retry.RecoveryCallback; +import org.springframework.retry.RetryCallback; +import org.springframework.retry.RetryContext; +import org.springframework.retry.support.RetryTemplate; +import org.springframework.util.Assert; +import org.springframework.util.StringUtils; +import org.springframework.xd.dirt.integration.bus.AbstractBusPropertiesAccessor; +import org.springframework.xd.dirt.integration.bus.Binding; +import org.springframework.xd.dirt.integration.bus.BusProperties; +import org.springframework.xd.dirt.integration.bus.EmbeddedHeadersMessageConverter; +import org.springframework.xd.dirt.integration.bus.MessageBus; +import org.springframework.xd.dirt.integration.bus.MessageBusSupport; +import org.springframework.xd.dirt.integration.bus.MessageValues; +import org.springframework.xd.dirt.integration.bus.XdHeaders; +import org.springframework.xd.dirt.integration.bus.serializer.MultiTypeCodec; + +/** + * A {@link MessageBus} implementation backed by Redis. + * @author Mark Fisher + * @author Gary Russell + * @author David Turanski + * @author Jennifer Hickey + */ +public class RedisMessageBus extends MessageBusSupport implements DisposableBean { + + private static final String ERROR_HEADER = "errorKey"; + + private static final SpelExpressionParser parser = new SpelExpressionParser(); + + private final String[] headersToMap; + + /** + * Retry only. + */ + private static final Set SUPPORTED_PUBSUB_CONSUMER_PROPERTIES = new SetBuilder() + .addAll(CONSUMER_STANDARD_PROPERTIES) + .addAll(CONSUMER_RETRY_PROPERTIES) + .build(); + + /** + * Retry + concurrency. + */ + private static final Set SUPPORTED_NAMED_CONSUMER_PROPERTIES = new SetBuilder() + .addAll(CONSUMER_STANDARD_PROPERTIES) + .addAll(CONSUMER_RETRY_PROPERTIES) + .add(BusProperties.CONCURRENCY) + .build(); + + /** + * Named + partitioning. + */ + private static final Set SUPPORTED_CONSUMER_PROPERTIES = new SetBuilder() + .addAll(SUPPORTED_NAMED_CONSUMER_PROPERTIES) + .add(BusProperties.PARTITION_INDEX) + .build(); + + /** + * Retry + concurrency (request). + */ + private static final Set SUPPORTED_REPLYING_CONSUMER_PROPERTIES = new SetBuilder() + // request + .addAll(CONSUMER_STANDARD_PROPERTIES) + .addAll(CONSUMER_RETRY_PROPERTIES) + .add(BusProperties.CONCURRENCY) + .build(); + + /** + * None. + */ + private static final Set SUPPORTED_PUBSUB_PRODUCER_PROPERTIES = PRODUCER_STANDARD_PROPERTIES; + + /** + * None. + */ + private static final Set SUPPORTED_NAMED_PRODUCER_PROPERTIES = PRODUCER_STANDARD_PROPERTIES; + + /** + * Partitioning. + */ + private static final Set SUPPORTED_PRODUCER_PROPERTIES = new SetBuilder() + .addAll(PRODUCER_PARTITIONING_PROPERTIES) + .addAll(PRODUCER_STANDARD_PROPERTIES) + .add(BusProperties.DIRECT_BINDING_ALLOWED) + .build(); + + /** + * Retry, concurrency (reply). + */ + private static final Set SUPPORTED_REQUESTING_PRODUCER_PROPERTIES = new SetBuilder() + // reply + .addAll(CONSUMER_RETRY_PROPERTIES) + .add(BusProperties.CONCURRENCY) + .build(); + + private final RedisConnectionFactory connectionFactory; + + private final EmbeddedHeadersMessageConverter embeddedHeadersMessageConverter = new + EmbeddedHeadersMessageConverter(); + + private final RedisQueueOutboundChannelAdapter errorAdapter; + + public RedisMessageBus(RedisConnectionFactory connectionFactory, MultiTypeCodec codec) { + this(connectionFactory, codec, new String[0]); + } + + public RedisMessageBus(RedisConnectionFactory connectionFactory, MultiTypeCodec codec, + String... headersToMap) { + Assert.notNull(connectionFactory, "connectionFactory must not be null"); + Assert.notNull(codec, "codec must not be null"); + this.connectionFactory = connectionFactory; + setCodec(codec); + this.errorAdapter = new RedisQueueOutboundChannelAdapter( + parser.parseExpression("headers['" + ERROR_HEADER + "']"), connectionFactory); + if (headersToMap != null && headersToMap.length > 0) { + String[] combinedHeadersToMap = + Arrays.copyOfRange(XdHeaders.STANDARD_HEADERS, 0, XdHeaders.STANDARD_HEADERS.length + + headersToMap.length); + System.arraycopy(headersToMap, 0, combinedHeadersToMap, XdHeaders.STANDARD_HEADERS.length, + headersToMap.length); + this.headersToMap = combinedHeadersToMap; + } + else { + this.headersToMap = XdHeaders.STANDARD_HEADERS; + } + } + + @Override + protected void onInit() { + this.errorAdapter.setIntegrationEvaluationContext(this.evaluationContext); + } + + @Override + public void bindConsumer(final String name, MessageChannel moduleInputChannel, Properties properties) { + if (name.startsWith(P2P_NAMED_CHANNEL_TYPE_PREFIX)) { + validateConsumerProperties(name, properties, SUPPORTED_NAMED_CONSUMER_PROPERTIES); + } + else { + validateConsumerProperties(name, properties, SUPPORTED_CONSUMER_PROPERTIES); + } + RedisPropertiesAccessor accessor = new RedisPropertiesAccessor(properties); + String queueName = "queue." + name; + int partitionIndex = accessor.getPartitionIndex(); + if (partitionIndex >= 0) { + queueName += "-" + partitionIndex; + } + MessageProducerSupport adapter = createInboundAdapter(accessor, queueName); + doRegisterConsumer(name, name + (partitionIndex >= 0 ? "-" + partitionIndex : ""), moduleInputChannel, adapter, + accessor); + bindExistingProducerDirectlyIfPossible(name, moduleInputChannel); + } + + private MessageProducerSupport createInboundAdapter(RedisPropertiesAccessor accessor, String queueName) { + MessageProducerSupport adapter; + int concurrency = accessor.getConcurrency(this.defaultConcurrency); + concurrency = concurrency > 0 ? concurrency : 1; + if (concurrency == 1) { + RedisQueueMessageDrivenEndpoint single = new RedisQueueMessageDrivenEndpoint(queueName, + this.connectionFactory); + single.setBeanFactory(getBeanFactory()); + single.setSerializer(null); + adapter = single; + } + else { + adapter = new CompositeRedisQueueMessageDrivenEndpoint(queueName, concurrency); + } + return adapter; + } + + @Override + public void bindPubSubConsumer(final String name, MessageChannel moduleInputChannel, + Properties properties) { + if (logger.isInfoEnabled()) { + logger.info("declaring pubsub for inbound: " + name); + } + validateConsumerProperties(name, properties, SUPPORTED_PUBSUB_CONSUMER_PROPERTIES); + RedisInboundChannelAdapter adapter = new RedisInboundChannelAdapter(this.connectionFactory); + adapter.setBeanFactory(this.getBeanFactory()); + adapter.setSerializer(null); + adapter.setTopics(applyPubSub(name)); + doRegisterConsumer(name, name, moduleInputChannel, adapter, new RedisPropertiesAccessor(properties)); + } + + private void doRegisterConsumer(String bindingName, String channelName, MessageChannel moduleInputChannel, + MessageProducerSupport adapter, RedisPropertiesAccessor properties) { + DirectChannel bridgeToModuleChannel = new DirectChannel(); + bridgeToModuleChannel.setBeanFactory(this.getBeanFactory()); + bridgeToModuleChannel.setBeanName(channelName + ".bridge"); + MessageChannel bridgeInputChannel = addRetryIfNeeded(channelName, bridgeToModuleChannel, properties); + adapter.setOutputChannel(bridgeInputChannel); + adapter.setBeanName("inbound." + bindingName); + adapter.afterPropertiesSet(); + Binding consumerBinding = Binding.forConsumer(bindingName, adapter, moduleInputChannel, properties); + addBinding(consumerBinding); + ReceivingHandler convertingBridge = new ReceivingHandler(); + convertingBridge.setOutputChannel(moduleInputChannel); + convertingBridge.setBeanName(channelName + ".bridge.handler"); + convertingBridge.afterPropertiesSet(); + bridgeToModuleChannel.subscribe(convertingBridge); + consumerBinding.start(); + } + + /** + * If retry is enabled, wrap the bridge channel in another that will invoke send() within the scope of a retry + * template. + * @param name The name. + * @param bridgeToModuleChannel The channel. + * @param properties The properties. + * @return The channel, or a wrapper. + */ + private MessageChannel addRetryIfNeeded(final String name, final DirectChannel bridgeToModuleChannel, + RedisPropertiesAccessor properties) { + final RetryTemplate retryTemplate = buildRetryTemplateIfRetryEnabled(properties); + if (retryTemplate == null) { + return bridgeToModuleChannel; + } + else { + DirectChannel channel = new DirectChannel() { + + @Override + protected boolean doSend(final Message message, final long timeout) { + try { + return retryTemplate.execute(new RetryCallback() { + + @Override + public Boolean doWithRetry(RetryContext context) throws Exception { + return bridgeToModuleChannel.send(message, timeout); + } + + }, new RecoveryCallback() { + + /** + * Send the failed message to 'ERRORS:[name]'. + */ + @Override + public Boolean recover(RetryContext context) throws Exception { + logger.error( + "Failed to deliver message; retries exhausted; message sent to queue 'ERRORS:" + + name + "' " + context.getLastThrowable()); + errorAdapter.handleMessage(getMessageBuilderFactory().fromMessage(message) + .setHeader(ERROR_HEADER, "ERRORS:" + name) + .build()); + return true; + } + + }); + } + catch (Exception e) { + logger.error("Failed to deliver message", e); + return false; + } + } + + }; + channel.setBeanName(name + ".bridge"); + return channel; + } + } + + @Override + public void bindProducer(final String name, MessageChannel moduleOutputChannel, + Properties properties) { + Assert.isInstanceOf(SubscribableChannel.class, moduleOutputChannel); + if (name.startsWith(P2P_NAMED_CHANNEL_TYPE_PREFIX)) { + validateProducerProperties(name, properties, SUPPORTED_NAMED_PRODUCER_PROPERTIES); + } + else { + validateProducerProperties(name, properties, SUPPORTED_PRODUCER_PROPERTIES); + } + RedisPropertiesAccessor accessor = new RedisPropertiesAccessor(properties); + if (!bindNewProducerDirectlyIfPossible(name, (SubscribableChannel) moduleOutputChannel, accessor)) { + String partitionKeyExtractorClass = accessor.getPartitionKeyExtractorClass(); + Expression partitionKeyExpression = accessor.getPartitionKeyExpression(); + RedisQueueOutboundChannelAdapter queue; + String queueName = "queue." + name; + if (partitionKeyExpression == null && !StringUtils.hasText(partitionKeyExtractorClass)) { + queue = new RedisQueueOutboundChannelAdapter(queueName, this.connectionFactory); + } + else { + queue = new RedisQueueOutboundChannelAdapter( + parser.parseExpression(buildPartitionRoutingExpression(queueName)), this.connectionFactory); + } + queue.setIntegrationEvaluationContext(this.evaluationContext); + queue.setBeanFactory(this.getBeanFactory()); + queue.afterPropertiesSet(); + doRegisterProducer(name, moduleOutputChannel, queue, accessor); + } + } + + @Override + public void bindPubSubProducer(final String name, MessageChannel moduleOutputChannel, + Properties properties) { + validateProducerProperties(name, properties, SUPPORTED_PUBSUB_PRODUCER_PROPERTIES); + RedisPublishingMessageHandler topic = new RedisPublishingMessageHandler(connectionFactory); + topic.setBeanFactory(this.getBeanFactory()); + topic.setTopic(applyPubSub(name)); + topic.afterPropertiesSet(); + doRegisterProducer(name, moduleOutputChannel, topic, new RedisPropertiesAccessor(properties)); + } + + private void doRegisterProducer(final String name, MessageChannel moduleOutputChannel, MessageHandler delegate, + RedisPropertiesAccessor properties) { + this.doRegisterProducer(name, moduleOutputChannel, delegate, null, properties); + } + + private void doRegisterProducer(final String name, MessageChannel moduleOutputChannel, MessageHandler delegate, + String replyTo, RedisPropertiesAccessor properties) { + Assert.isInstanceOf(SubscribableChannel.class, moduleOutputChannel); + MessageHandler handler = new SendingHandler(delegate, replyTo, properties); + EventDrivenConsumer consumer = new EventDrivenConsumer((SubscribableChannel) moduleOutputChannel, handler); + consumer.setBeanFactory(this.getBeanFactory()); + consumer.setBeanName("outbound." + name); + consumer.afterPropertiesSet(); + Binding producerBinding = Binding.forProducer(name, moduleOutputChannel, consumer, properties); + addBinding(producerBinding); + producerBinding.start(); + } + + @Override + public void bindRequestor(String name, MessageChannel requests, MessageChannel replies, + Properties properties) { + if (logger.isInfoEnabled()) { + logger.info("binding requestor: " + name); + } + Assert.isInstanceOf(SubscribableChannel.class, requests); + validateProducerProperties(name, properties, SUPPORTED_REQUESTING_PRODUCER_PROPERTIES); + RedisQueueOutboundChannelAdapter queue = new RedisQueueOutboundChannelAdapter("queue." + applyRequests(name), + this.connectionFactory); + queue.setBeanFactory(this.getBeanFactory()); + queue.afterPropertiesSet(); + String replyQueueName = name + ".replies." + this.getIdGenerator().generateId(); + RedisPropertiesAccessor accessor = new RedisPropertiesAccessor(properties); + this.doRegisterProducer(name, requests, queue, replyQueueName, accessor); + MessageProducerSupport adapter = createInboundAdapter(accessor, replyQueueName); + this.doRegisterConsumer(name, name, replies, adapter, accessor); + } + + @Override + public void bindReplier(String name, MessageChannel requests, MessageChannel replies, + Properties properties) { + if (logger.isInfoEnabled()) { + logger.info("binding replier: " + name); + } + validateConsumerProperties(name, properties, SUPPORTED_REPLYING_CONSUMER_PROPERTIES); + RedisPropertiesAccessor accessor = new RedisPropertiesAccessor(properties); + MessageProducerSupport adapter = createInboundAdapter(accessor, "queue." + applyRequests(name)); + this.doRegisterConsumer(name, name, requests, adapter, accessor); + + RedisQueueOutboundChannelAdapter replyQueue = new RedisQueueOutboundChannelAdapter( + RedisMessageBus.parser.parseExpression("headers['" + XdHeaders.REPLY_TO + "']"), + this.connectionFactory); + replyQueue.setBeanFactory(this.getBeanFactory()); + replyQueue.setIntegrationEvaluationContext(this.evaluationContext); + replyQueue.afterPropertiesSet(); + this.doRegisterProducer(name, replies, replyQueue, accessor); + } + + @Override + public void destroy() { + stopBindings(); + } + + private class SendingHandler extends AbstractMessageHandler { + + private final MessageHandler delegate; + + private final String replyTo; + + private final PartitioningMetadata partitioningMetadata; + + + private SendingHandler(MessageHandler delegate, String replyTo, RedisPropertiesAccessor properties) { + this.delegate = delegate; + this.replyTo = replyTo; + this.partitioningMetadata = new PartitioningMetadata(properties, properties.getNextModuleCount()); + this.setBeanFactory(RedisMessageBus.this.getBeanFactory()); + } + + @Override + protected void handleMessageInternal(Message message) throws Exception { + MessageValues transformed = serializePayloadIfNecessary(message); + + if (replyTo != null) { + transformed.put(XdHeaders.REPLY_TO, this.replyTo); + } + if (this.partitioningMetadata.isPartitionedModule()) { + + transformed.put(PARTITION_HEADER, determinePartition(message, this.partitioningMetadata)); + } + + byte[] messageToSend = embeddedHeadersMessageConverter.embedHeaders(transformed, + RedisMessageBus.this.headersToMap); + delegate.handleMessage(MessageBuilder.withPayload(messageToSend).copyHeaders(transformed).build()); + } + + } + + private class ReceivingHandler extends AbstractReplyProducingMessageHandler { + + public ReceivingHandler() { + super(); + this.setBeanFactory(RedisMessageBus.this.getBeanFactory()); + } + + @SuppressWarnings("unchecked") + @Override + protected Object handleRequestMessage(Message requestMessage) { + MessageValues theRequestMessage; + try { + theRequestMessage = embeddedHeadersMessageConverter.extractHeaders((Message) requestMessage, true); + } + catch (Exception e) { + logger.error(EmbeddedHeadersMessageConverter.decodeExceptionMessage(requestMessage), e); + theRequestMessage = new MessageValues(requestMessage); + } + return deserializePayloadIfNecessary(theRequestMessage).toMessage(getMessageBuilderFactory()); + } + + @Override + protected boolean shouldCopyRequestHeaders() { + // prevent returned message from being copied in superclass + return false; + } + } + + private static class RedisPropertiesAccessor extends AbstractBusPropertiesAccessor { + + public RedisPropertiesAccessor(Properties properties) { + super(properties); + } + + } + + /** + * Provides concurrency by creating a list of message-driven endpoints. + */ + private class CompositeRedisQueueMessageDrivenEndpoint extends MessageProducerSupport { + + private final List consumers = new + ArrayList(); + + public CompositeRedisQueueMessageDrivenEndpoint(String queueName, int concurrency) { + for (int i = 0; i < concurrency; i++) { + RedisQueueMessageDrivenEndpoint adapter = new RedisQueueMessageDrivenEndpoint(queueName, + connectionFactory); + adapter.setBeanFactory(RedisMessageBus.this.getBeanFactory()); + adapter.setSerializer(null); + adapter.setBeanName("inbound." + queueName + "." + i); + this.consumers.add(adapter); + } + this.setBeanFactory(RedisMessageBus.this.getBeanFactory()); + } + + @Override + protected void onInit() { + for (RedisQueueMessageDrivenEndpoint consumer : consumers) { + consumer.afterPropertiesSet(); + } + } + + @Override + protected void doStart() { + for (RedisQueueMessageDrivenEndpoint consumer : consumers) { + consumer.start(); + } + } + + @Override + protected void doStop() { + for (RedisQueueMessageDrivenEndpoint consumer : consumers) { + consumer.stop(); + } + } + + @Override + public void setOutputChannel(MessageChannel outputChannel) { + for (RedisQueueMessageDrivenEndpoint consumer : consumers) { + consumer.setOutputChannel(outputChannel); + } + } + + @Override + public void setErrorChannel(MessageChannel errorChannel) { + for (RedisQueueMessageDrivenEndpoint consumer : consumers) { + consumer.setErrorChannel(errorChannel); + } + } + + } + +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-redis/src/main/java/org/springframework/xd/dirt/integration/redis/package-info.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-redis/src/main/java/org/springframework/xd/dirt/integration/redis/package-info.java new file mode 100644 index 000000000..75376dfa5 --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-redis/src/main/java/org/springframework/xd/dirt/integration/redis/package-info.java @@ -0,0 +1,5 @@ +/** + * This package contains an implementation of the {@link org.springframework.xd.dirt.integration.bus.MessageBus} for Redis. + */ + +package org.springframework.xd.dirt.integration.redis; diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-redis/src/main/resources/META-INF/spring-xd/bus/redis-bus.xml b/spring-cloud-streams-bindings/spring-cloud-streams-binding-redis/src/main/resources/META-INF/spring-xd/bus/redis-bus.xml new file mode 100644 index 000000000..a7ffb79e7 --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-redis/src/main/resources/META-INF/spring-xd/bus/redis-bus.xml @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-redis/src/test/java/org/springframework/xd/dirt/integration/bus/redis/RedisMessageBusTests.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-redis/src/test/java/org/springframework/xd/dirt/integration/bus/redis/RedisMessageBusTests.java new file mode 100644 index 000000000..68fab3d8b --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-redis/src/test/java/org/springframework/xd/dirt/integration/bus/redis/RedisMessageBusTests.java @@ -0,0 +1,395 @@ +/* + * Copyright 2013-2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.xd.dirt.integration.bus.redis; + +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.instanceOf; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.mock; + +import java.util.Arrays; +import java.util.Collection; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.TimeUnit; + +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; + +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.data.redis.serializer.StringRedisSerializer; +import org.springframework.expression.Expression; +import org.springframework.integration.channel.DirectChannel; +import org.springframework.integration.endpoint.AbstractEndpoint; +import org.springframework.integration.redis.inbound.RedisQueueMessageDrivenEndpoint; +import org.springframework.integration.test.util.TestUtils; +import org.springframework.messaging.support.GenericMessage; +import org.springframework.retry.support.RetryTemplate; +import org.springframework.xd.dirt.integration.bus.Binding; +import org.springframework.xd.dirt.integration.bus.BusProperties; +import org.springframework.xd.dirt.integration.bus.EmbeddedHeadersMessageConverter; +import org.springframework.xd.dirt.integration.bus.MessageBus; +import org.springframework.xd.dirt.integration.bus.PartitionCapableBusTests; +import org.springframework.xd.dirt.integration.bus.Spy; +import org.springframework.xd.dirt.integration.redis.RedisMessageBus; +import org.springframework.xd.dirt.integration.redis.RedisTestSupport; + +/** + * @author Gary Russell + */ +public class RedisMessageBusTests extends PartitionCapableBusTests { + + @Rule + public RedisTestSupport redisAvailableRule = new RedisTestSupport(); + + private RedisTemplate redisTemplate; + + private static final EmbeddedHeadersMessageConverter embeddedHeadersMessageConverter = + new EmbeddedHeadersMessageConverter(); + + @Override + protected MessageBus getMessageBus() { + if (testMessageBus == null) { + testMessageBus = new RedisTestMessageBus(redisAvailableRule.getResource(), getCodec()); + } + return testMessageBus; + } + + @Override + protected boolean usesExplicitRouting() { + return true; + } + + @Override + public void testSendAndReceivePubSub() throws Exception { + + TimeUnit.SECONDS.sleep(2); //TODO remove timing issue + + super.testSendAndReceivePubSub(); + } + + @Before + public void setup() { + createTemplate().boundListOps("queue.direct.0").trim(1, 0); + } + + @Test + public void testConsumerProperties() throws Exception { + MessageBus bus = getMessageBus(); + Properties properties = new Properties(); + properties.put("maxAttempts", "1"); // disable retry + bus.bindConsumer("props.0", new DirectChannel(), properties); + @SuppressWarnings("unchecked") + List bindings = TestUtils.getPropertyValue(bus, "messageBus.bindings", List.class); + assertEquals(1, bindings.size()); + AbstractEndpoint endpoint = bindings.get(0).getEndpoint(); + assertThat(endpoint, instanceOf(RedisQueueMessageDrivenEndpoint.class)); + assertSame(DirectChannel.class, TestUtils.getPropertyValue(endpoint, "outputChannel").getClass()); + bus.unbindConsumers("props.0"); + assertEquals(0, bindings.size()); + + properties.put("backOffInitialInterval", "2000"); + properties.put("backOffMaxInterval", "20000"); + properties.put("backOffMultiplier", "5.0"); + properties.put("concurrency", "2"); + properties.put("maxAttempts", "23"); + properties.put("partitionIndex", 0); + + bus.bindConsumer("props.0", new DirectChannel(), properties); + assertEquals(1, bindings.size()); + endpoint = bindings.get(0).getEndpoint(); + verifyConsumer(endpoint); + + try { + bus.bindPubSubConsumer("dummy", null, properties); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertThat(e.getMessage(), allOf( + containsString("RedisMessageBus does not support consumer properties: "), + containsString("partitionIndex"), + containsString("concurrency"), + containsString(" for dummy."))); + } + try { + bus.bindConsumer("queue:dummy", null, properties); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertEquals("RedisMessageBus does not support consumer property: partitionIndex for queue:dummy.", + e.getMessage()); + } + + bus.unbindConsumers("props.0"); + assertEquals(0, bindings.size()); + } + + @Test + public void testProducerProperties() throws Exception { + MessageBus bus = getMessageBus(); + bus.bindProducer("props.0", new DirectChannel(), null); + @SuppressWarnings("unchecked") + List bindings = TestUtils.getPropertyValue(bus, "messageBus.bindings", List.class); + assertEquals(1, bindings.size()); + AbstractEndpoint endpoint = bindings.get(0).getEndpoint(); + assertEquals( + "queue.props.0", + TestUtils.getPropertyValue(endpoint, "handler.delegate.queueNameExpression", Expression.class).getExpressionString()); + bus.unbindProducers("props.0"); + assertEquals(0, bindings.size()); + + Properties properties = new Properties(); + properties.put("partitionKeyExpression", "'foo'"); + properties.put("partitionKeyExtractorClass", "foo"); + properties.put("partitionSelectorExpression", "0"); + properties.put("partitionSelectorClass", "foo"); + properties.put(BusProperties.NEXT_MODULE_COUNT, "1"); + + bus.bindProducer("props.0", new DirectChannel(), properties); + assertEquals(1, bindings.size()); + endpoint = bindings.get(0).getEndpoint(); + assertEquals( + "'queue.props.0-' + headers['partition']", + TestUtils.getPropertyValue(endpoint, "handler.delegate.queueNameExpression", Expression.class).getExpressionString()); + + try { + bus.bindPubSubProducer("dummy", null, properties); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertThat(e.getMessage(), allOf( + containsString("RedisMessageBus does not support producer properties: "), + containsString("partitionSelectorExpression"), + containsString("partitionKeyExtractorClass"), + containsString("partitionKeyExpression"), + containsString("partitionSelectorClass"))); + assertThat(e.getMessage(), containsString("for dummy.")); + } + try { + bus.bindProducer("queue:dummy", new DirectChannel(), properties); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertThat(e.getMessage(), allOf( + containsString("RedisMessageBus does not support producer properties: "), + containsString("partitionSelectorExpression"), + containsString("partitionKeyExtractorClass"), + containsString("partitionKeyExpression"), + containsString("partitionSelectorClass"))); + assertThat(e.getMessage(), containsString("for queue:dummy.")); + } + + bus.unbindProducers("props.0"); + assertEquals(0, bindings.size()); + } + + @Test + public void testRequestReplyRequestorProperties() throws Exception { + MessageBus bus = getMessageBus(); + Properties properties = new Properties(); + + properties.put("backOffInitialInterval", "2000"); + properties.put("backOffMaxInterval", "20000"); + properties.put("backOffMultiplier", "5.0"); + properties.put("concurrency", "2"); + properties.put("maxAttempts", "23"); + + bus.bindRequestor("props.0", new DirectChannel(), new DirectChannel(), properties); + @SuppressWarnings("unchecked") + List bindings = TestUtils.getPropertyValue(bus, "messageBus.bindings", List.class); + + assertEquals(2, bindings.size()); + AbstractEndpoint endpoint = bindings.get(0).getEndpoint(); // producer + assertEquals( + "queue.props.0.requests", + TestUtils.getPropertyValue(endpoint, "handler.delegate.queueNameExpression", Expression.class).getExpressionString()); + + endpoint = bindings.get(1).getEndpoint(); // consumer + verifyConsumer(endpoint); + + properties.put("partitionKeyExpression", "'foo'"); + properties.put("partitionKeyExtractorClass", "foo"); + properties.put("partitionSelectorExpression", "0"); + properties.put("partitionSelectorClass", "foo"); + properties.put("partitionIndex", "0"); + try { + bus.bindRequestor("dummy", new DirectChannel(), new DirectChannel(), properties); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertThat(e.getMessage(), allOf( + containsString("RedisMessageBus does not support producer properties: "), + containsString("partitionSelectorExpression"), + containsString("partitionKeyExtractorClass"), + containsString("partitionKeyExpression"), + containsString("partitionSelectorClass"))); + assertThat(e.getMessage(), allOf(containsString("partitionIndex"), containsString("for dummy."))); + } + + bus.unbindConsumers("props.0"); + bus.unbindProducers("props.0"); + assertEquals(0, bindings.size()); + } + + @Test + public void testRequestReplyReplierProperties() throws Exception { + MessageBus bus = getMessageBus(); + Properties properties = new Properties(); + + properties.put("backOffInitialInterval", "2000"); + properties.put("backOffMaxInterval", "20000"); + properties.put("backOffMultiplier", "5.0"); + properties.put("concurrency", "2"); + properties.put("maxAttempts", "23"); + + bus.bindReplier("props.0", new DirectChannel(), new DirectChannel(), properties); + @SuppressWarnings("unchecked") + List bindings = TestUtils.getPropertyValue(bus, "messageBus.bindings", List.class); + + assertEquals(2, bindings.size()); + AbstractEndpoint endpoint = bindings.get(1).getEndpoint(); // producer + assertEquals( + "headers['replyTo']", + TestUtils.getPropertyValue(endpoint, "handler.delegate.queueNameExpression", Expression.class).getExpressionString()); + + endpoint = bindings.get(0).getEndpoint(); // consumer + verifyConsumer(endpoint); + + properties.put("partitionKeyExpression", "'foo'"); + properties.put("partitionKeyExtractorClass", "foo"); + properties.put("partitionSelectorExpression", "0"); + properties.put("partitionSelectorClass", "foo"); + properties.put(BusProperties.NEXT_MODULE_COUNT, "1"); + properties.put("partitionIndex", "0"); + try { + bus.bindReplier("dummy", new DirectChannel(), new DirectChannel(), properties); + fail("Expected exception"); + } + catch (IllegalArgumentException e) { + assertThat(e.getMessage(), allOf( + containsString("RedisMessageBus does not support consumer properties: "), + containsString("partitionSelectorExpression"), + containsString("partitionKeyExtractorClass"), + containsString("partitionKeyExpression"), + containsString("partitionSelectorClass"))); + assertThat(e.getMessage(), allOf(containsString("partitionIndex"), containsString("for dummy."))); + } + + bus.unbindConsumers("props.0"); + bus.unbindProducers("props.0"); + assertEquals(0, bindings.size()); + } + + private void verifyConsumer(AbstractEndpoint endpoint) { + assertThat(endpoint.getClass().getName(), containsString("CompositeRedisQueueMessageDrivenEndpoint")); + assertEquals(2, TestUtils.getPropertyValue(endpoint, "consumers", Collection.class).size()); + DirectChannel channel = TestUtils.getPropertyValue( + TestUtils.getPropertyValue(endpoint, "consumers", List.class).get(0), + "outputChannel", DirectChannel.class); + assertThat( + channel.getClass().getName(), containsString("RedisMessageBus$")); // retry wrapper + assertThat( + TestUtils.getPropertyValue(TestUtils.getPropertyValue(endpoint, "consumers", List.class).get(1), + "outputChannel").getClass().getName(), containsString("RedisMessageBus$")); // retry wrapper + RetryTemplate retry = TestUtils.getPropertyValue(channel, "val$retryTemplate", RetryTemplate.class); + assertEquals(23, TestUtils.getPropertyValue(retry, "retryPolicy.maxAttempts")); + assertEquals(2000L, TestUtils.getPropertyValue(retry, "backOffPolicy.initialInterval")); + assertEquals(20000L, TestUtils.getPropertyValue(retry, "backOffPolicy.maxInterval")); + assertEquals(5.0, TestUtils.getPropertyValue(retry, "backOffPolicy.multiplier")); + } + + @Test + public void testRetryFail() { + MessageBus bus = getMessageBus(); + DirectChannel channel = new DirectChannel(); + bus.bindProducer("retry.0", channel, null); + Properties props = new Properties(); + props.put("maxAttempts", 2); + props.put("backOffInitialInterval", 100); + props.put("backOffMultiplier", "1.0"); + bus.bindConsumer("retry.0", new DirectChannel(), props); // no subscriber + channel.send(new GenericMessage("foo")); + RedisTemplate template = createTemplate(); + Object rightPop = template.boundListOps("ERRORS:retry.0").rightPop(5, TimeUnit.SECONDS); + assertNotNull(rightPop); + assertThat(new String((byte[]) rightPop), containsString("foo")); + } + + @Test + public void testMoreHeaders() { + RedisMessageBus bus = new RedisMessageBus(mock(RedisConnectionFactory.class), getCodec(), "foo", "bar"); + Collection headers = Arrays.asList(TestUtils.getPropertyValue(bus, "headersToMap", String[].class)); + assertEquals(10, headers.size()); + assertTrue(headers.contains("foo")); + assertTrue(headers.contains("bar")); + } + + private RedisTemplate createTemplate() { + if (this.redisTemplate != null) { + return this.redisTemplate; + } + RedisTemplate template = new RedisTemplate(); + template.setConnectionFactory(this.redisAvailableRule.getResource()); + template.setKeySerializer(new StringRedisSerializer()); + template.setEnableDefaultSerializer(false); + template.afterPropertiesSet(); + this.redisTemplate = template; + return template; + } + + @Override + protected String getEndpointRouting(AbstractEndpoint endpoint) { + return TestUtils.getPropertyValue(endpoint, "handler.delegate.queueNameExpression", Expression.class).getExpressionString(); + } + + @Override + protected String getPubSubEndpointRouting(AbstractEndpoint endpoint) { + return TestUtils.getPropertyValue(endpoint, "handler.delegate.topicExpression", Expression.class).getExpressionString(); + } + + @Override + public Spy spyOn(final String queue) { + final RedisTemplate template = createTemplate(); + return new Spy() { + + @Override + public Object receive(boolean expectNull) throws Exception { + byte[] bytes = (byte[]) template.boundListOps("queue." + queue).rightPop(50, TimeUnit.MILLISECONDS); + if (bytes == null) { + return null; + } + bytes = (byte[]) embeddedHeadersMessageConverter.extractHeaders(new GenericMessage(bytes), false).getPayload(); + return new String(bytes, "UTF-8"); + } + + }; + } + + @Override + protected void busBindUnbindLatency() throws InterruptedException { + Thread.sleep(3000); // needed for Redis see INT-3442 + } + +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-redis/src/test/java/org/springframework/xd/dirt/integration/bus/redis/RedisTestMessageBus.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-redis/src/test/java/org/springframework/xd/dirt/integration/bus/redis/RedisTestMessageBus.java new file mode 100644 index 000000000..cd6b87dba --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-redis/src/test/java/org/springframework/xd/dirt/integration/bus/redis/RedisTestMessageBus.java @@ -0,0 +1,73 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.xd.dirt.integration.bus.redis; + +import org.springframework.context.support.GenericApplicationContext; +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.integration.channel.DefaultHeaderChannelRegistry; +import org.springframework.integration.context.IntegrationContextUtils; +import org.springframework.integration.support.DefaultMessageBuilderFactory; +import org.springframework.integration.support.utils.IntegrationUtils; +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler; +import org.springframework.xd.dirt.integration.bus.AbstractTestMessageBus; +import org.springframework.xd.dirt.integration.bus.serializer.MultiTypeCodec; +import org.springframework.xd.dirt.integration.redis.RedisMessageBus; + + +/** + * Test support class for {@link RedisMessageBus}. + * + * @author Ilayaperumal Gopinathan + * @author Gary Russell + */ +public class RedisTestMessageBus extends AbstractTestMessageBus { + + private StringRedisTemplate template; + + public RedisTestMessageBus(RedisConnectionFactory connectionFactory) { + template = new StringRedisTemplate(connectionFactory); + } + + public RedisTestMessageBus(RedisConnectionFactory connectionFactory, MultiTypeCodec codec) { + RedisMessageBus messageBus = new RedisMessageBus(connectionFactory, codec); + GenericApplicationContext context = new GenericApplicationContext(); + context.getBeanFactory().registerSingleton(IntegrationUtils.INTEGRATION_MESSAGE_BUILDER_FACTORY_BEAN_NAME, + new DefaultMessageBuilderFactory()); + DefaultHeaderChannelRegistry channelRegistry = new DefaultHeaderChannelRegistry(); + channelRegistry.setReaperDelay(Long.MAX_VALUE); + ThreadPoolTaskScheduler taskScheduler = new ThreadPoolTaskScheduler(); + taskScheduler.afterPropertiesSet(); + channelRegistry.setTaskScheduler(taskScheduler); + context.getBeanFactory().registerSingleton( + IntegrationContextUtils.INTEGRATION_HEADER_CHANNEL_REGISTRY_BEAN_NAME, + channelRegistry); + context.refresh(); + messageBus.setApplicationContext(context); + setMessageBus(messageBus); + template = new StringRedisTemplate(connectionFactory); + } + + @Override + public void cleanup() { + if (!queues.isEmpty()) { + for (String queue : queues) { + template.delete(queue); + } + } + } +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-redis/src/test/java/org/springframework/xd/dirt/integration/redis/AbstractRedisSerializerTests.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-redis/src/test/java/org/springframework/xd/dirt/integration/redis/AbstractRedisSerializerTests.java new file mode 100644 index 000000000..cd94eafd1 --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-redis/src/test/java/org/springframework/xd/dirt/integration/redis/AbstractRedisSerializerTests.java @@ -0,0 +1,153 @@ +/* + * Copyright 2002-2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.springframework.xd.dirt.integration.redis; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.util.Date; +import java.util.HashMap; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeSet; + +import org.joda.time.DateTime; +import org.junit.Before; +import org.junit.Test; + +import org.springframework.data.redis.serializer.RedisSerializer; +import org.springframework.xd.tuple.Tuple; +import org.springframework.xd.tuple.TupleBuilder; + +import com.fasterxml.jackson.annotation.JsonCreator; +import com.fasterxml.jackson.annotation.JsonProperty; + +/** + * @author David Turanski + * + */ +public abstract class AbstractRedisSerializerTests { + + private RedisSerializer serializer; + + protected abstract RedisSerializer getSerializer(); + + @Before + public void setUp() { + serializer = getSerializer(); + } + + @Test + public void testRandomObjectSerialization() { + Foo foo = new Foo("hello"); + byte[] bytes = serializer.serialize(foo); + Object obj = serializer.deserialize(bytes); + assertTrue(obj instanceof Foo); + assertEquals("hello", ((Foo) obj).bar); + } + + @Test + public void testDateSerialization() { + Date d = new Date(); + byte[] bytes = serializer.serialize(d); + Date obj = (Date) serializer.deserialize(bytes); + assertEquals(d, obj); + } + + @Test + public void testDateTimeSerialization() { + DateTime d = new DateTime(); + byte[] bytes = serializer.serialize(d); + DateTime obj = (DateTime) serializer.deserialize(bytes); + assertEquals(d, obj); + } + + @Test + public void testStringSerialization() { + String s = new String("hello"); + byte[] bytes = serializer.serialize(s); + Object obj = serializer.deserialize(bytes); + assertEquals(s, obj); + } + + @Test + public void testLongSerialization() { + byte[] bytes = serializer.serialize(100L); + long obj = (Long) serializer.deserialize(bytes); + assertEquals(100, obj); + } + + @Test + public void testFloatSerialization() { + byte[] bytes = serializer.serialize(99.9f); + double obj = (Double) serializer.deserialize(bytes); + assertEquals(99.9, obj, 0.1); + } + + @Test + public void testBooleanSerialization() { + byte[] bytes = serializer.serialize(true); + boolean obj = (Boolean) serializer.deserialize(bytes); + assertTrue(obj); + } + + @Test + public void testMapSerialization() { + Map map = new HashMap(); + map.put("foo", "bar"); + byte[] bytes = serializer.serialize(map); + Map obj = (Map) serializer.deserialize(bytes); + assertEquals("bar", obj.get("foo")); + } + + @Test + public void testListSerialization() { + List list = new LinkedList(); + list.add("foo"); + byte[] bytes = serializer.serialize(list); + List obj = (List) serializer.deserialize(bytes); + assertEquals("foo", obj.get(0)); + } + + @Test + public void testSetSerialization() { + Set set = new TreeSet(); + set.add("foo"); + byte[] bytes = serializer.serialize(set); + Set obj = (Set) serializer.deserialize(bytes); + assertEquals("foo", obj.iterator().next()); + } + + @Test + public void testTupleSerialization() { + Tuple t = TupleBuilder.tuple().of("foo", "bar"); + byte[] bytes = serializer.serialize(t); + + Tuple obj = (Tuple) serializer.deserialize(bytes); + assertEquals("bar", obj.getString("foo")); + } + + public static class Foo { + + @JsonCreator + public Foo(@JsonProperty("bar") String val) { + bar = val; + } + + public String bar; + } + +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-redis/src/test/java/org/springframework/xd/dirt/integration/redis/RedisPublishingMessageHandlerTests.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-redis/src/test/java/org/springframework/xd/dirt/integration/redis/RedisPublishingMessageHandlerTests.java new file mode 100644 index 000000000..78772dc56 --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-redis/src/test/java/org/springframework/xd/dirt/integration/redis/RedisPublishingMessageHandlerTests.java @@ -0,0 +1,144 @@ +/* + * Copyright 2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.xd.dirt.integration.redis; + +import static org.junit.Assert.assertEquals; + +import java.util.Collections; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; + +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.data.redis.listener.ChannelTopic; +import org.springframework.data.redis.listener.RedisMessageListenerContainer; +import org.springframework.data.redis.listener.Topic; +import org.springframework.data.redis.listener.adapter.MessageListenerAdapter; +import org.springframework.data.redis.serializer.GenericToStringSerializer; +import org.springframework.data.redis.serializer.RedisSerializer; +import org.springframework.data.redis.serializer.StringRedisSerializer; +import org.springframework.integration.redis.outbound.RedisPublishingMessageHandler; +import org.springframework.integration.support.MessageBuilder; +import org.springframework.xd.dirt.integration.bus.BusTestUtils; + +/** + * Temporary copy of SI RedisPublishingMessageHandlerTests that adds tests that publish messages with data types other + * than String + * + * @author Mark Fisher + * @author Jennifer Hickey + * @author Gary Russell + */ +public class RedisPublishingMessageHandlerTests { + + private static final String TOPIC = "si.test.channel"; + + private static final int NUM_MESSAGES = 10; + + private RedisConnectionFactory connectionFactory; + + private RedisMessageListenerContainer container; + + private CountDownLatch latch = new CountDownLatch(NUM_MESSAGES); + + @Rule + public RedisTestSupport redisAvailableRule = new RedisTestSupport(); + + @Before + public void setUp() { + this.connectionFactory = redisAvailableRule.getResource(); + } + + @Test + public void testWithDefaultSerializer() throws Exception { + setupListener(new StringRedisSerializer()); + final RedisPublishingMessageHandler handler = new RedisPublishingMessageHandler(connectionFactory); + handler.setBeanFactory(BusTestUtils.MOCK_BF); + handler.setTopic(TOPIC); + handler.afterPropertiesSet(); + for (int i = 0; i < NUM_MESSAGES; i++) { + handler.handleMessage(MessageBuilder.withPayload("test-" + i).build()); + } + latch.await(3, TimeUnit.SECONDS); + assertEquals(0, latch.getCount()); + container.stop(); + } + + @Test + public void testWithNoSerializer() throws Exception { + setupListener(null); + final RedisPublishingMessageHandler handler = new RedisPublishingMessageHandler(connectionFactory); + handler.setBeanFactory(BusTestUtils.MOCK_BF); + handler.setTopic(TOPIC); + handler.afterPropertiesSet(); + for (int i = 0; i < NUM_MESSAGES; i++) { + handler.handleMessage(MessageBuilder.withPayload(new String("test-" + i).getBytes()).build()); + } + latch.await(3, TimeUnit.SECONDS); + assertEquals(0, latch.getCount()); + container.stop(); + } + + @Test + public void testWithCustomSerializer() throws Exception { + GenericToStringSerializer serializer = new GenericToStringSerializer(Long.class); + setupListener(serializer); + final RedisPublishingMessageHandler handler = new RedisPublishingMessageHandler(connectionFactory); + handler.setBeanFactory(BusTestUtils.MOCK_BF); + handler.setTopic(TOPIC); + handler.setSerializer(serializer); + handler.afterPropertiesSet(); + for (long i = 0; i < NUM_MESSAGES; i++) { + handler.handleMessage(MessageBuilder.withPayload(i).build()); + } + latch.await(3, TimeUnit.SECONDS); + assertEquals(0, latch.getCount()); + container.stop(); + } + + private void setupListener(RedisSerializer listenerSerializer) throws InterruptedException { + MessageListenerAdapter listener = new MessageListenerAdapter(); + listener.setDelegate(new Listener(latch)); + listener.setSerializer(listenerSerializer); + listener.afterPropertiesSet(); + + this.container = new RedisMessageListenerContainer(); + container.setConnectionFactory(connectionFactory); + container.afterPropertiesSet(); + container.addMessageListener(listener, Collections. singletonList(new ChannelTopic(TOPIC))); + container.start(); + Thread.sleep(1000); + } + + private static class Listener { + + private final CountDownLatch latch; + + private Listener(CountDownLatch latch) { + this.latch = latch; + } + + @SuppressWarnings("unused") + public void handleMessage(Object s) { + this.latch.countDown(); + } + } + +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-redis/src/test/java/org/springframework/xd/dirt/integration/redis/RedisQueueInboundChannelAdapterTests.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-redis/src/test/java/org/springframework/xd/dirt/integration/redis/RedisQueueInboundChannelAdapterTests.java new file mode 100644 index 000000000..002d2ae85 --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-redis/src/test/java/org/springframework/xd/dirt/integration/redis/RedisQueueInboundChannelAdapterTests.java @@ -0,0 +1,216 @@ +/* + * Copyright 2002-2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.xd.dirt.integration.redis; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.BlockingDeque; +import java.util.concurrent.LinkedBlockingDeque; +import java.util.concurrent.TimeUnit; + +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; + +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.data.redis.serializer.GenericToStringSerializer; +import org.springframework.data.redis.serializer.JdkSerializationRedisSerializer; +import org.springframework.data.redis.serializer.RedisSerializer; +import org.springframework.data.redis.serializer.SerializationException; +import org.springframework.data.redis.serializer.StringRedisSerializer; +import org.springframework.integration.channel.DirectChannel; +import org.springframework.integration.redis.inbound.RedisQueueMessageDrivenEndpoint; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageHandler; +import org.springframework.messaging.MessagingException; +import org.springframework.messaging.support.GenericMessage; +import org.springframework.xd.dirt.integration.bus.BusTestUtils; + +/** + * Integration test of {@link RedisQueueInboundChannelAdapter} + * + * @author Jennifer Hickey + */ +public class RedisQueueInboundChannelAdapterTests { + + private static final String QUEUE_NAME = "inboundadaptertest"; + + private RedisConnectionFactory connectionFactory; + + private final BlockingDeque messages = new LinkedBlockingDeque(99); + + private RedisQueueMessageDrivenEndpoint adapter; + + @Rule + public RedisTestSupport redisAvailableRule = new RedisTestSupport(); + + private String currentQueueName; + + @Before + public void setUp() { + messages.clear(); + this.connectionFactory = redisAvailableRule.getResource(); + DirectChannel outputChannel = new DirectChannel(); + outputChannel.setBeanFactory(BusTestUtils.MOCK_BF); + outputChannel.subscribe(new TestMessageHandler()); + + this.currentQueueName = QUEUE_NAME + ":" + System.nanoTime(); + + adapter = new RedisQueueMessageDrivenEndpoint(currentQueueName, connectionFactory); + adapter.setBeanFactory(BusTestUtils.MOCK_BF); + adapter.setOutputChannel(outputChannel); + } + + @After + public void tearDown() { + adapter.stop(); + connectionFactory.getConnection().del(currentQueueName.getBytes()); + } + + @Test + public void testDefaultPayloadSerializer() throws Exception { + RedisTemplate template = new RedisTemplate(); + template.setConnectionFactory(connectionFactory); + template.setKeySerializer(new StringRedisSerializer()); + template.afterPropertiesSet(); + + adapter.afterPropertiesSet(); + adapter.start(); + + template.boundListOps(currentQueueName).rightPush("message1"); + @SuppressWarnings("unchecked") + Message message = (Message) messages.poll(1, TimeUnit.SECONDS); + assertNotNull(message); + assertEquals("message1", message.getPayload()); + } + + @Test + public void testDefaultMsgSerializer() throws Exception { + RedisTemplate> template = new RedisTemplate>(); + template.setKeySerializer(new StringRedisSerializer()); + template.setValueSerializer(new JdkSerializationRedisSerializer()); + template.setConnectionFactory(connectionFactory); + template.afterPropertiesSet(); + + adapter.setExpectMessage(true); + adapter.afterPropertiesSet(); + adapter.start(); + + Map headers = new HashMap(); + headers.put("header1", "foo"); + template.boundListOps(currentQueueName).rightPush(new GenericMessage("message2", headers)); + @SuppressWarnings("unchecked") + Message message = (Message) messages.poll(1, TimeUnit.SECONDS); + assertEquals("message2", message.getPayload()); + assertEquals("foo", message.getHeaders().get("header1")); + } + + @SuppressWarnings("unchecked") + @Test + public void testNoSerializer() throws Exception { + RedisTemplate template = new RedisTemplate(); + template.setEnableDefaultSerializer(false); + template.setKeySerializer(new StringRedisSerializer()); + template.setConnectionFactory(connectionFactory); + template.afterPropertiesSet(); + + adapter.setSerializer(null); + adapter.afterPropertiesSet(); + adapter.start(); + + template.boundListOps(currentQueueName).rightPush("message3".getBytes()); + Message message = (Message) messages.poll(1, TimeUnit.SECONDS); + assertEquals("message3", new String(message.getPayload())); + } + + @Test(expected = IllegalArgumentException.class) + public void testNoSerializerNoExtractPayload() throws Exception { + RedisTemplate template = new RedisTemplate(); + template.setEnableDefaultSerializer(false); + template.setKeySerializer(new StringRedisSerializer()); + template.setConnectionFactory(connectionFactory); + template.afterPropertiesSet(); + + adapter.setSerializer(null); + adapter.setExpectMessage(true); + adapter.afterPropertiesSet(); + adapter.start(); + } + + @Test + public void testCustomPayloadSerializer() throws Exception { + RedisTemplate template = new RedisTemplate(); + template.setKeySerializer(new StringRedisSerializer()); + template.setValueSerializer(new GenericToStringSerializer(Long.class)); + template.setConnectionFactory(connectionFactory); + template.afterPropertiesSet(); + + adapter.setSerializer(new GenericToStringSerializer(Long.class)); + adapter.afterPropertiesSet(); + adapter.start(); + + template.boundListOps(currentQueueName).rightPush(5l); + @SuppressWarnings("unchecked") + Message message = (Message) messages.poll(1, TimeUnit.SECONDS); + assertEquals(5L, (long) message.getPayload()); + } + + @Test + public void testCustomMessageSerializer() throws Exception { + RedisTemplate> template = new RedisTemplate>(); + template.setKeySerializer(new StringRedisSerializer()); + template.setValueSerializer(new TestMessageSerializer()); + template.setConnectionFactory(connectionFactory); + template.afterPropertiesSet(); + + adapter.setSerializer(new TestMessageSerializer()); + adapter.setExpectMessage(true); + adapter.afterPropertiesSet(); + adapter.start(); + + template.boundListOps(currentQueueName).rightPush(new GenericMessage(10l)); + @SuppressWarnings("unchecked") + Message message = (Message) messages.poll(1, TimeUnit.SECONDS); + assertEquals(10L, (long) message.getPayload()); + } + + private class TestMessageHandler implements MessageHandler { + + @Override + public void handleMessage(Message message) throws MessagingException { + messages.add(message); + } + } + + private class TestMessageSerializer implements RedisSerializer> { + + @Override + public byte[] serialize(Message t) throws SerializationException { + return "Foo".getBytes(); + } + + @Override + public Message deserialize(byte[] bytes) throws SerializationException { + return new GenericMessage(10l); + } + } +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-redis/src/test/java/org/springframework/xd/dirt/integration/redis/RedisQueueOutboundChannelAdapterTests.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-redis/src/test/java/org/springframework/xd/dirt/integration/redis/RedisQueueOutboundChannelAdapterTests.java new file mode 100644 index 000000000..42e5bc94a --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-redis/src/test/java/org/springframework/xd/dirt/integration/redis/RedisQueueOutboundChannelAdapterTests.java @@ -0,0 +1,171 @@ +/* + * Copyright 2002-2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.xd.dirt.integration.redis; + +import static org.junit.Assert.assertEquals; + +import java.util.HashMap; +import java.util.Map; + +import org.junit.After; +import org.junit.Before; +import org.junit.Rule; +import org.junit.Test; + +import org.springframework.data.redis.connection.RedisConnectionFactory; +import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.data.redis.core.StringRedisTemplate; +import org.springframework.data.redis.serializer.GenericToStringSerializer; +import org.springframework.data.redis.serializer.JdkSerializationRedisSerializer; +import org.springframework.data.redis.serializer.RedisSerializer; +import org.springframework.data.redis.serializer.SerializationException; +import org.springframework.data.redis.serializer.StringRedisSerializer; +import org.springframework.integration.redis.outbound.RedisQueueOutboundChannelAdapter; +import org.springframework.messaging.Message; +import org.springframework.messaging.support.GenericMessage; +import org.springframework.xd.dirt.integration.bus.BusTestUtils; + +/** + * Integration test of {@link RedisQueueOutboundChannelAdapter} + * + * @author Jennifer Hickey + * @author Gary Russell + */ +public class RedisQueueOutboundChannelAdapterTests { + + private static final String QUEUE_NAME = "outboundadaptertest"; + + private RedisConnectionFactory connectionFactory; + + private RedisQueueOutboundChannelAdapter adapter; + + @Rule + public RedisTestSupport redisAvailableRule = new RedisTestSupport(); + + @Before + public void setUp() { + this.connectionFactory = redisAvailableRule.getResource(); + adapter = new RedisQueueOutboundChannelAdapter(QUEUE_NAME, connectionFactory); + adapter.setBeanFactory(BusTestUtils.MOCK_BF); + } + + @After + public void tearDown() { + connectionFactory.getConnection().del(QUEUE_NAME.getBytes()); + } + + @Test + public void testDefaultPayloadSerializer() throws Exception { + StringRedisTemplate template = new StringRedisTemplate(connectionFactory); + template.afterPropertiesSet(); + + adapter.afterPropertiesSet(); + adapter.handleMessage(new GenericMessage("message1")); + assertEquals("message1", template.boundListOps(QUEUE_NAME).rightPop()); + } + + @Test + public void testDefaultMsgSerializer() throws Exception { + RedisTemplate> template = new RedisTemplate>(); + template.setKeySerializer(new StringRedisSerializer()); + template.setValueSerializer(new JdkSerializationRedisSerializer()); + template.setConnectionFactory(connectionFactory); + template.afterPropertiesSet(); + + adapter.setExtractPayload(false); + adapter.afterPropertiesSet(); + + Map headers = new HashMap(); + headers.put("header1", "foo"); + adapter.handleMessage(new GenericMessage("message2", headers)); + Message message = template.boundListOps(QUEUE_NAME).rightPop(); + assertEquals("message2", message.getPayload()); + assertEquals("foo", message.getHeaders().get("header1")); + } + + @Test + public void testNoSerializer() throws Exception { + RedisTemplate template = new RedisTemplate(); + template.setEnableDefaultSerializer(false); + template.setKeySerializer(new StringRedisSerializer()); + template.setConnectionFactory(connectionFactory); + template.afterPropertiesSet(); + + adapter.afterPropertiesSet(); + + adapter.handleMessage(new GenericMessage("message3".getBytes())); + byte[] value = template.boundListOps(QUEUE_NAME).rightPop(); + assertEquals("message3", new String(value)); + } + + @Test(expected = IllegalArgumentException.class) + public void testNoSerializerNoExtractPayload() throws Exception { + RedisTemplate template = new RedisTemplate(); + template.setEnableDefaultSerializer(false); + template.setKeySerializer(new StringRedisSerializer()); + template.setConnectionFactory(connectionFactory); + template.afterPropertiesSet(); + + adapter.setSerializer(null); + adapter.setExtractPayload(false); + adapter.afterPropertiesSet(); + } + + @Test + public void testCustomPayloadSerializer() throws Exception { + RedisTemplate template = new RedisTemplate(); + template.setKeySerializer(new StringRedisSerializer()); + template.setValueSerializer(new GenericToStringSerializer(Long.class)); + template.setConnectionFactory(connectionFactory); + template.afterPropertiesSet(); + + adapter.setSerializer(new GenericToStringSerializer(Long.class)); + adapter.afterPropertiesSet(); + + adapter.handleMessage(new GenericMessage(5l)); + assertEquals(Long.valueOf(5), template.boundListOps(QUEUE_NAME).rightPop()); + } + + @Test + public void testCustomMessageSerializer() throws Exception { + RedisTemplate> template = new RedisTemplate>(); + template.setKeySerializer(new StringRedisSerializer()); + template.setValueSerializer(new TestMessageSerializer()); + template.setConnectionFactory(connectionFactory); + template.afterPropertiesSet(); + + adapter.setSerializer(new TestMessageSerializer()); + adapter.setExtractPayload(false); + adapter.afterPropertiesSet(); + + Message message = template.boundListOps(QUEUE_NAME).rightPop(); + assertEquals(10l, message.getPayload()); + } + + private class TestMessageSerializer implements RedisSerializer> { + + @Override + public byte[] serialize(Message t) throws SerializationException { + return "Foo".getBytes(); + } + + @Override + public Message deserialize(byte[] bytes) throws SerializationException { + return new GenericMessage(10l); + } + } +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-redis/src/test/java/org/springframework/xd/dirt/integration/redis/RedisTestSupport.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-redis/src/test/java/org/springframework/xd/dirt/integration/redis/RedisTestSupport.java new file mode 100644 index 000000000..ba576a4af --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-redis/src/test/java/org/springframework/xd/dirt/integration/redis/RedisTestSupport.java @@ -0,0 +1,47 @@ +/* + * Copyright 2002-2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.xd.dirt.integration.redis; + +import org.junit.Rule; + +import org.springframework.data.redis.connection.jedis.JedisConnectionFactory; +import org.springframework.xd.test.AbstractExternalResourceTestSupport; + +/** + * JUnit {@link Rule} that detects the fact that a Redis server is running on localhost. + * + * @author Gary Russell + * @author Eric Bottard + */ +public class RedisTestSupport extends AbstractExternalResourceTestSupport { + + public RedisTestSupport() { + super("REDIS"); + } + + @Override + protected void obtainResource() throws Exception { + resource = new JedisConnectionFactory(); + resource.afterPropertiesSet(); + resource.getConnection().close(); + } + + @Override + protected void cleanupResource() throws Exception { + resource.destroy(); + } +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/pom.xml b/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/pom.xml new file mode 100644 index 000000000..96d001dea --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/pom.xml @@ -0,0 +1,47 @@ + + + 4.0.0 + + spring-cloud-streams-binding-spi + jar + spring-cloud-streams-binding-spi + SPI for binding implementations + + + org.springframework.cloud + spring-cloud-streams-bindings-parent + 1.0.0.BUILD-SNAPSHOT + + + + UTF-8 + + + + + org.springframework.cloud + spring-cloud-streams-codec + + + org.springframework + spring-web + ${spring-framework.version} + + + org.springframework.retry + spring-retry + 1.1.0.RELEASE + + + com.fasterxml.jackson.core + jackson-databind + 2.4.5 + + + org.apache.httpcomponents + httpclient + 4.3.6 + + + diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/AbstractBusPropertiesAccessor.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/AbstractBusPropertiesAccessor.java new file mode 100644 index 000000000..f6cc88199 --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/AbstractBusPropertiesAccessor.java @@ -0,0 +1,379 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.xd.dirt.integration.bus; + +import java.util.Properties; + +import org.springframework.expression.Expression; +import org.springframework.expression.spel.standard.SpelExpressionParser; +import org.springframework.util.StringUtils; + + +/** + * Base class for bus-specific property accessors; common properties + * are defined here. + * + * @author Gary Russell + */ +public abstract class AbstractBusPropertiesAccessor implements BusProperties { + + private static final SpelExpressionParser spelExpressionParser = new SpelExpressionParser(); + + private final Properties properties; + + public AbstractBusPropertiesAccessor(Properties properties) { + if (properties == null) { + this.properties = new Properties(); + } + else { + this.properties = properties; + } + } + + /** + * Return the underlying properties object. + * @return The properties. + */ + public Properties getProperties() { + return properties; + } + + /** + * Return the property for the key, or null if it doesn't exist. + * @param key The property. + * @return The key. + */ + public String getProperty(String key) { + return this.properties.getProperty(key); + } + + /** + * Return the property for the key, or the default value if the + * property doesn't exist. + * @param key The key. + * @param defaultValue The default value. + * @return The property or default value. + */ + public String getProperty(String key, String defaultValue) { + return this.properties.getProperty(key, defaultValue); + } + + /** + * Return the property for the key, or the default value if the + * property doesn't exist. + * @param key The key. + * @param defaultValue The default value. + * @return The property or default value. + */ + public boolean getProperty(String key, boolean defaultValue) { + String property = this.properties.getProperty(key); + if (property != null) { + return Boolean.parseBoolean(property); + } + else { + return defaultValue; + } + } + + /** + * Return the property for the key, or the default value if the + * property doesn't exist. + * @param key The key. + * @param defaultValue The default value. + * @return The property or default value. + */ + public int getProperty(String key, int defaultValue) { + String property = this.properties.getProperty(key); + if (property != null) { + return Integer.parseInt(property); + } + else { + return defaultValue; + } + } + + /** + * Return the property for the key, or the default value if the + * property doesn't exist. + * @param key The key. + * @param defaultValue The default value. + * @return The property or default value. + */ + public long getProperty(String key, long defaultValue) { + String property = this.properties.getProperty(key); + if (property != null) { + return Long.parseLong(property); + } + else { + return defaultValue; + } + } + + /** + * Return the property for the key, or the default value if the + * property doesn't exist. + * @param key The key. + * @param defaultValue The default value. + * @return The property or default value. + */ + public double getProperty(String key, double defaultValue) { + String property = properties.getProperty(key); + if (property != null) { + return Double.parseDouble(property); + } + else { + return defaultValue; + } + } + + /** + * Return the 'concurrency' property or the default value. + * The meaning of concurrency depends on the bus implementation. + * @param defaultValue The default value. + * @return The property or default value. + */ + public int getConcurrency(int defaultValue) { + return getProperty(CONCURRENCY, defaultValue); + } + + /** + * Return the 'maxConcurrency' property or the default value. + * The meaning of maxConcurrency depends on the bus implementation. + * @param defaultValue The default value. + * @return The property or default value. + */ + public int getMaxConcurrency(int defaultValue) { + return getProperty(MAX_CONCURRENCY, defaultValue); + } + + // Retry properties + + /** + * Return the 'maxAttempts' property or the default value. + * This is used in the retry template's SimpleRetryPolicy + * in buses that support retry. + * @param defaultValue The default value. + * @return The property or default value. + */ + public int getMaxAttempts(int defaultValue) { + return getProperty(MAX_ATTEMPTS, defaultValue); + } + + /** + * Return the 'backOffInitialInterval' property or the default value. + * This is used in the retry template's ExponentialBackOffPolicy + * in buses that support retry. + * @param defaultValue The default value. + * @return The property or default value. + */ + public long getBackOffInitialInterval(long defaultValue) { + return getProperty(BACK_OFF_INITIAL_INTERVAL, defaultValue); + } + + /** + * Return the 'backOffMultiplier' property or the default value. + * This is used in the retry template's ExponentialBackOffPolicy + * in buses that support retry. + * @param defaultValue The default value. + * @return The property or default value. + */ + public double getBackOffMultiplier(double defaultValue) { + return getProperty(BACK_OFF_MULTIPLIER, defaultValue); + } + + /** + * Return the 'backOffMaxInterval' property or the default value. + * This is used in the retry template's ExponentialBackOffPolicy + * in buses that support retry. + * @param defaultValue The default value. + * @return The property or default value. + */ + public long getBackOffMaxInterval(long defaultValue) { + return getProperty(BACK_OFF_MAX_INTERVAL, defaultValue); + } + + // Partitioning + + /** + * A class name for extracting partition keys from messages. + * @return The class name, + */ + public String getPartitionKeyExtractorClass() { + return getProperty(PARTITION_KEY_EXTRACTOR_CLASS); + } + + /** + * The expression to determine the partition key, evaluated against the + * message as the root object. + * @return The key. + */ + public Expression getPartitionKeyExpression() { + String partionKeyExpression = getProperty(PARTITION_KEY_EXPRESSION); + Expression expression = null; + if (partionKeyExpression != null) { + expression = spelExpressionParser.parseExpression(partionKeyExpression); + } + return expression; + } + + /** + * A class name for calculating a partition from a key. + * @return The class name, + */ + public String getPartitionSelectorClass() { + return getProperty(PARTITION_SELECTOR_CLASS); + } + + /** + * The expression evaluated against the partition key to determine + * the partition to which the message will be sent. The result should + * be an integer that will subsequently be mod'd with the module's + * partition count. + * @return The expression. + */ + public Expression getPartitionSelectorExpression() { + String partionSelectorExpression = getProperty(PARTITION_SELECTOR_EXPRESSION); + Expression expression = null; + if (partionSelectorExpression != null) { + expression = spelExpressionParser.parseExpression(partionSelectorExpression); + } + return expression; + } + + /** + * The sequence number for this module. + * + * @return the sequence number. + */ + public int getSequence() { + return getProperty(SEQUENCE, 1); + } + + /** + * The module count. + * + * @return the module count. + */ + public int getCount() { + return getProperty(COUNT, 1); + } + + /** + * The next module count for non-sink modules + * @return the next module count + */ + public int getNextModuleCount() { + return getProperty(NEXT_MODULE_COUNT, 1); + } + + /** + * The partition index that this consumer supports. + * @return The partition index. + */ + public int getPartitionIndex() { + return getProperty(PARTITION_INDEX, -1); + } + + // Direct Binding + + /** + * If true, the bus can attempt a direct binding. + */ + public boolean isDirectBindingAllowed() { + return getProperty(DIRECT_BINDING_ALLOWED, false); + } + + // Batching + + /** + * If true, enable batching. + * @param defaultValue the default value. + * @return the property or default value. + */ + public boolean isBatchingEnabled(boolean defaultValue) { + return getProperty(BATCHING_ENABLED, defaultValue); + } + + /** + * The batch size. + * @param defaultValue the default value. + * @return the property or default value. + */ + public int getBatchSize(int defaultValue) { + return getProperty(BATCH_SIZE, defaultValue); + } + + /** + * The batch buffer limit. + * @param defaultValue the default value. + * @return the property or default value. + */ + public int geteBatchBufferLimit(int defaultValue) { + return getProperty(BATCH_BUFFER_LIMIT, defaultValue); + } + + /** + * The batch timeout. + * @param defaultValue the default value. + * @return the property or default value. + */ + public long getBatchTimeout(long defaultValue) { + return getProperty(BATCH_TIMEOUT, defaultValue); + } + + /** + * If true, messages will be compressed. + * @param defaultValue the default value. + * @return the property or default value. + */ + public boolean isCompress(boolean defaultValue) { + return getProperty(COMPRESS, defaultValue); + } + + /** + * If true, subscriptions to taps/topics will be durable. + * @param defaultValue the default value. + * @return the property or default value. + */ + public boolean isDurable(boolean defaultValue) { + return getProperty(DURABLE, defaultValue); + } + + // Utility methods + + /** + * Convert a comma-delimited String property to a String[] if + * present, or return the default value. + * @param value The property value. + * @param defaultValue The default value. + * @return The converted property or default value. + */ + protected String[] asStringArray(String value, String[] defaultValue) { + if (StringUtils.hasText(value)) { + return StringUtils.commaDelimitedListToStringArray(value); + } + else { + return defaultValue; + } + } + + @Override + public String toString() { + return this.properties.toString(); + } + +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/Binding.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/Binding.java new file mode 100644 index 000000000..78cd86730 --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/Binding.java @@ -0,0 +1,119 @@ +/* + * Copyright 2013-2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.xd.dirt.integration.bus; + +import org.springframework.context.Lifecycle; +import org.springframework.integration.endpoint.AbstractEndpoint; +import org.springframework.messaging.MessageChannel; +import org.springframework.util.Assert; + +/** + * Represents a binding between a module's channel and an adapter endpoint that connects to the MessageBus. The binding + * could be for a consumer or a producer. A consumer binding represents a connection from an adapter on the bus to a + * module's input channel. A producer binding represents a connection from a module's output channel to an adapter on + * the bus. + * + * @author Jennifer Hickey + * @author Mark Fisher + * @author Gary Russell + */ +public class Binding implements Lifecycle { + + public static final String PRODUCER = "producer"; + + public static final String CONSUMER = "consumer"; + + public static final String DIRECT = "direct"; + + private final String name; + + private final MessageChannel channel; + + private final AbstractEndpoint endpoint; + + private final String type; + + private final AbstractBusPropertiesAccessor properties; + + private Binding(String name, MessageChannel channel, AbstractEndpoint endpoint, String type, + AbstractBusPropertiesAccessor properties) { + Assert.notNull(channel, "channel must not be null"); + Assert.notNull(endpoint, "endpoint must not be null"); + this.name = name; + this.channel = channel; + this.endpoint = endpoint; + this.type = type; + this.properties = properties; + } + + public static Binding forConsumer(String name, AbstractEndpoint adapterFromBus, MessageChannel moduleInputChannel, + AbstractBusPropertiesAccessor properties) { + return new Binding(name, moduleInputChannel, adapterFromBus, CONSUMER, properties); + } + + public static Binding forProducer(String name, MessageChannel moduleOutputChannel, AbstractEndpoint adapterToBus, + AbstractBusPropertiesAccessor properties) { + return new Binding(name, moduleOutputChannel, adapterToBus, PRODUCER, properties); + } + + public static Binding forDirectProducer(String name, MessageChannel moduleOutputChannel, + AbstractEndpoint adapter, AbstractBusPropertiesAccessor properties) { + return new Binding(name, moduleOutputChannel, adapter, DIRECT, properties); + } + + public String getName() { + return name; + } + + public MessageChannel getChannel() { + return channel; + } + + public AbstractEndpoint getEndpoint() { + return endpoint; + } + + public String getType() { + return type; + } + + public AbstractBusPropertiesAccessor getPropertiesAccessor() { + return properties; + } + + @Override + public void start() { + endpoint.start(); + } + + @Override + public void stop() { + endpoint.stop(); + } + + @Override + public boolean isRunning() { + return endpoint.isRunning(); + } + + @Override + public String toString() { + return type + " Binding [name=" + name + ", channel=" + channel + ", endpoint=" + endpoint.getComponentName() + + "]"; + } + +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/BusCleaner.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/BusCleaner.java new file mode 100644 index 000000000..b6b2bc5b6 --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/BusCleaner.java @@ -0,0 +1,39 @@ +/* + * Copyright 2015 the original author or authors. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.xd.dirt.integration.bus; + +import java.util.List; +import java.util.Map; + + +/** + * Interface for implementations that perform cleanup for message buses. + * + * @author Gary Russell + * @since 1.2 + */ +public interface BusCleaner { + + /** + * Clean up all resources for the supplied stream/job. + * @param entity the stream or job; may be terminated with a simple wild card '*', in which + * case all streams with names starting with the characters before the '*' will be cleaned. + * @param isJob true if the entity is a job. + * @return a map of lists of resources removed. + */ + Map> clean(String entity, boolean isJob); + +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/BusProperties.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/BusProperties.java new file mode 100644 index 000000000..8c9881c34 --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/BusProperties.java @@ -0,0 +1,144 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.xd.dirt.integration.bus; + + +/** + * Common bus properties. + * + * @author Gary Russell + */ +public interface BusProperties { + + /** + * The retry back off initial interval. + */ + public static final String BACK_OFF_INITIAL_INTERVAL = "backOffInitialInterval"; + + /** + * The retry back off max interval. + */ + public static final String BACK_OFF_MAX_INTERVAL = "backOffMaxInterval"; + + /** + * The retry back off multiplier. + */ + public static final String BACK_OFF_MULTIPLIER = "backOffMultiplier"; + + /** + * The minimum number of concurrent deliveries. + */ + public static final String CONCURRENCY = "concurrency"; + + /** + * The maximum delivery attempts when a delivery fails. + */ + public static final String MAX_ATTEMPTS = "maxAttempts"; + + /** + * The maximum number of concurrent deliveries. + */ + public static final String MAX_CONCURRENCY = "maxConcurrency"; + + /** + * The sequence index of the module. + * In a partitioned stream, it is identical to the partition index. + */ + public static final String SEQUENCE = "sequence"; + + /** + * The number of consumers, i.e. module instances in the stream. + * In a partitioned stream, it is identical to the partition count. + */ + public static final String COUNT = "count"; + + /** + * The consumer's partition index. + */ + public static final String PARTITION_INDEX = "partitionIndex"; + + /** + * The partition key expression. + */ + public static final String PARTITION_KEY_EXPRESSION = "partitionKeyExpression"; + + /** + * The partition key class. + */ + public static final String PARTITION_KEY_EXTRACTOR_CLASS = "partitionKeyExtractorClass"; + + /** + * The partition selector class. + */ + public static final String PARTITION_SELECTOR_CLASS = "partitionSelectorClass"; + + /** + * The partition selector expression. + */ + public static final String PARTITION_SELECTOR_EXPRESSION = "partitionSelectorExpression"; + + /** + * If true, the bus will attempt to create a direct binding between the producer and consumer. + */ + public static final String DIRECT_BINDING_ALLOWED = "directBindingAllowed"; + + /** + * True if message batching is enabled. + */ + public static final String BATCHING_ENABLED = "batchingEnabled"; + + /** + * The batch size if batching is enabled. + */ + public static final String BATCH_SIZE = "batchSize"; + + /** + * The buffer limit if batching is enabled. + */ + public static final String BATCH_BUFFER_LIMIT = "batchBufferLimit"; + + /** + * The batch timeout if batching is enabled. + */ + public static final String BATCH_TIMEOUT = "batchTimeout"; + + /** + * For all non-terminal modules, the number of modules coming after this one, irrespective of partitioning. + */ + public static final String NEXT_MODULE_COUNT = "next.module.count"; + + /** + * For all non-terminal modules, the concurrency for module coming after this one. + */ + public static final String NEXT_MODULE_CONCURRENCY = "next.module.concurrency"; + + /** + * Compression enabled. + */ + public static final String COMPRESS = "compress"; + + /** + * Durable pub/sub consumer. + */ + public static final String DURABLE = "durableSubscription"; + + /** + * Minimum partition count, if the transport supports partitioning natively (e.g. Kafka) + */ + public static final String MIN_PARTITION_COUNT = "minPartitionCount"; + +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/BusUtils.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/BusUtils.java new file mode 100644 index 000000000..8c080ae53 --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/BusUtils.java @@ -0,0 +1,90 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.xd.dirt.integration.bus; + +import java.util.regex.Pattern; + +import org.springframework.util.Assert; +import org.springframework.util.StringUtils; + +/** + * Message Bus utilities. + * + * @author Gary Russell + */ +public class BusUtils { + + /** + * The delimiter between a group and index when constructing a bus consumer/producer. + */ + public static final String GROUP_INDEX_DELIMITER = "."; + + /** + * The prefix for the consumer/producer when creating a tap. + */ + public static final String TAP_CHANNEL_PREFIX = "tap:"; + + /** + * The prefix for the consumer/producer when creating a topic. + */ + public static final String TOPIC_CHANNEL_PREFIX = "topic:"; + + public static final Pattern PUBSUB_NAMED_CHANNEL_PATTERN = Pattern.compile("[^.]+\\.(tap|topic):"); + + public static String addGroupToPubSub(String group, String inputChannelName) { + if (inputChannelName.startsWith(TAP_CHANNEL_PREFIX) + || inputChannelName.startsWith(TOPIC_CHANNEL_PREFIX)) { + inputChannelName = group + "." + inputChannelName; + } + return inputChannelName; + } + + public static String removeGroupFromPubSub(String name) { + if (PUBSUB_NAMED_CHANNEL_PATTERN.matcher(name).find()) { + return name.substring(name.indexOf(".") + 1); + } + else { + return name; + } + } + + /** + * Determine whether the provided channel name represents a pub/sub channel (i.e. topic or tap). + * @param channelName name of the channel to check + * @return true if pub/sub. + */ + public static boolean isChannelPubSub(String channelName) { + Assert.isTrue(StringUtils.hasText(channelName), "Channel name should not be empty/null."); + // Check if the channelName starts with tap: or topic: + return (channelName.startsWith(TAP_CHANNEL_PREFIX) || channelName.startsWith(TOPIC_CHANNEL_PREFIX)); + } + + /** + * Construct a pipe name from the group and index. + * @param group the group. + * @param index the index. + * @return the name. + */ + public static String constructPipeName(String group, int index) { + return group + GROUP_INDEX_DELIMITER + index; + } + + public static String constructTapPrefix(String group) { + return TAP_CHANNEL_PREFIX + "stream:" + group; + } + +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/EmbeddedHeadersMessageConverter.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/EmbeddedHeadersMessageConverter.java new file mode 100644 index 000000000..a238727d6 --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/EmbeddedHeadersMessageConverter.java @@ -0,0 +1,158 @@ +/* + * Copyright 2014-2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.xd.dirt.integration.bus; + +import java.io.UnsupportedEncodingException; +import java.nio.ByteBuffer; +import java.util.HashMap; +import java.util.Map; + +import javax.xml.bind.DatatypeConverter; + +import org.springframework.integration.IntegrationMessageHeaderAccessor; +import org.springframework.integration.support.json.Jackson2JsonObjectMapper; +import org.springframework.messaging.Message; + +/** + * Encodes requested headers into payload with format + * {@code 0xff, n(1), [ [lenHdr(1), hdr, lenValue(4), value] ... ]}. + * The 0xff indicates this new format; n is number of headers (max 255); for + * each header, the name length (1 byte) is followed by the name, followed by + * the value length (int) followed by the value (json). + *

+ * Previously, there was no leading 0xff; the value length was 1 byte and only + * String header values were supported (no JSON conversion). + * + * @author Eric Bottard + * @author Gary Russell + */ +public class EmbeddedHeadersMessageConverter { + + private final Jackson2JsonObjectMapper objectMapper = new Jackson2JsonObjectMapper(); + + public static String decodeExceptionMessage(Message requestMessage) { + return "Could not convert message: " + DatatypeConverter.printHexBinary((byte[]) requestMessage.getPayload()); + } + + + /** + * Return a new message where some of the original headers of {@code original} + * have been embedded into the new message payload. + */ + public byte[] embedHeaders(MessageValues original, String... headers) throws Exception { + byte[][] headerValues = new byte[headers.length][]; + int n = 0; + int headerCount = 0; + int headersLength = 0; + for (String header : headers) { + Object value = original.get(header) == null ? null + : original.get(header); + if (value != null) { + String json = this.objectMapper.toJson(value); + headerValues[n++] = json.getBytes("UTF-8"); + headerCount++; + headersLength += header.length() + json.length(); + } + else { + headerValues[n++] = null; + } + } + // 0xff, n(1), [ [lenHdr(1), hdr, lenValue(4), value] ... ] + byte[] newPayload = new byte[((byte[])original.getPayload()).length + headersLength + headerCount * 5 + 2]; + ByteBuffer byteBuffer = ByteBuffer.wrap(newPayload); + byteBuffer.put((byte) 0xff); // signal new format + byteBuffer.put((byte) headerCount); + for (int i = 0; i < headers.length; i++) { + if (headerValues[i] != null) { + byteBuffer.put((byte) headers[i].length()); + byteBuffer.put(headers[i].getBytes("UTF-8")); + byteBuffer.putInt(headerValues[i].length); + byteBuffer.put(headerValues[i]); + } + } + + byteBuffer.put((byte[])original.getPayload()); + return byteBuffer.array(); + } + + /** + * Return a message where headers, that were originally embedded into the payload, have been promoted + * back to actual headers. The new payload is now the original payload. + * + * @param message the message to extract headers + * @param copyRequestHeaders boolean value to specify if original headers should be copied + */ + public MessageValues extractHeaders(Message message, boolean copyRequestHeaders) throws Exception { + byte[] bytes = message.getPayload(); + ByteBuffer byteBuffer = ByteBuffer.wrap(bytes); + int headerCount = byteBuffer.get() & 0xff; + if (headerCount < 255) { + return oldExtractHeaders(byteBuffer, bytes, headerCount, message, copyRequestHeaders); + } + else { + headerCount = byteBuffer.get() & 0xff; + Map headers = new HashMap(); + for (int i = 0; i < headerCount; i++) { + int len = byteBuffer.get() & 0xff; + String headerName = new String(bytes, byteBuffer.position(), len, "UTF-8"); + byteBuffer.position(byteBuffer.position() + len); + len = byteBuffer.getInt(); + String headerValue = new String(bytes, byteBuffer.position(), len, "UTF-8"); + Object headerContent = this.objectMapper.fromJson(headerValue, Object.class); + headers.put(headerName, headerContent); + byteBuffer.position(byteBuffer.position() + len); + } + byte[] newPayload = new byte[byteBuffer.remaining()]; + byteBuffer.get(newPayload); + return buildMessageValues(message, newPayload, headers, copyRequestHeaders); + } + } + + private MessageValues oldExtractHeaders(ByteBuffer byteBuffer, byte[] bytes, int headerCount, + Message message, boolean copyRequestHeaders) + throws UnsupportedEncodingException { + Map headers = new HashMap(); + for (int i = 0; i < headerCount; i++) { + int len = byteBuffer.get(); + String headerName = new String(bytes, byteBuffer.position(), len, "UTF-8"); + byteBuffer.position(byteBuffer.position() + len); + len = byteBuffer.get() & 0xff; + String headerValue = new String(bytes, byteBuffer.position(), len, "UTF-8"); + byteBuffer.position(byteBuffer.position() + len); + if (IntegrationMessageHeaderAccessor.SEQUENCE_NUMBER.equals(headerName) + || IntegrationMessageHeaderAccessor.SEQUENCE_SIZE.equals(headerName)) { + headers.put(headerName, Integer.parseInt(headerValue)); + } + else { + headers.put(headerName, headerValue); + } + } + byte[] newPayload = new byte[byteBuffer.remaining()]; + byteBuffer.get(newPayload); + return buildMessageValues(message, newPayload, headers, copyRequestHeaders); + } + + private MessageValues buildMessageValues(Message message, byte[] payload, Map headers, + boolean copyRequestHeaders) { + MessageValues messageValues = new MessageValues(payload, headers); + if (copyRequestHeaders) { + messageValues.copyHeadersIfAbsent(message.getHeaders()); + } + return messageValues; + } + +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/MessageBus.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/MessageBus.java new file mode 100644 index 000000000..61118b9df --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/MessageBus.java @@ -0,0 +1,158 @@ +/* + * Copyright 2013-2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.springframework.xd.dirt.integration.bus; + +import java.util.Properties; + +import org.springframework.messaging.MessageChannel; + +/** + * A strategy interface used to bind a {@link MessageChannel} to a logical name. The name is intended to identify a + * logical consumer or producer of messages. This may be a queue, a channel adapter, another message channel, a Spring + * bean, etc. + * + * @author Mark Fisher + * @author David Turanski + * @author Gary Russell + * @author Jennifer Hickey + * @author Ilayaperumal Gopinathan + * @since 1.0 + */ +public interface MessageBus { + + /** + * Bind a message consumer on a p2p channel + * + * @param name the logical identity of the message source + * @param moduleInputChannel the channel bound as a consumer + * @param properties arbitrary String key/value pairs that will be used in the binding + */ + void bindConsumer(String name, MessageChannel moduleInputChannel, Properties properties); + + + /** + * Bind a message consumer on a pub/sub channel + * + * @param name the logical identity of the message source + * @param inputChannel the channel bound as a pub/sub consumer + * @param properties arbitrary String key/value pairs that will be used in the binding + */ + void bindPubSubConsumer(final String name, MessageChannel inputChannel, Properties properties); + + /** + * Bind a message producer on a p2p channel. + * + * @param name the logical identity of the message target + * @param moduleOutputChannel the channel bound as a producer + * @param properties arbitrary String key/value pairs that will be used in the binding + */ + void bindProducer(String name, MessageChannel moduleOutputChannel, Properties properties); + + + /** + * Bind a message producer on a pub/sub channel. + * + * @param name the logical identity of the message target + * @param outputChannel the channel bound as a producer + * @param properties arbitrary String key/value pairs that will be used in the binding + */ + void bindPubSubProducer(final String name, MessageChannel outputChannel, Properties properties); + + /** + * Unbind an inbound inter-module channel and stop any active components that use the channel. + * + * @param name the channel name + */ + void unbindConsumers(String name); + + /** + * Unbind an outbound inter-module channel and stop any active components that use the channel. + * + * @param name the channel name + */ + void unbindProducers(String name); + + /** + * Unbind a specific p2p or pub/sub message consumer + * + * @param name The logical identify of a message source + * @param channel The channel bound as a consumer + */ + void unbindConsumer(String name, MessageChannel channel); + + /** + * Unbind a specific p2p or pub/sub message producer + * + * @param name the logical identity of the message target + * @param channel the channel bound as a producer + */ + void unbindProducer(String name, MessageChannel channel); + + /** + * Bind a producer that expects async replies. To unbind, invoke unbindProducer() and unbindConsumer(). + * + * @param name The name of the requestor. + * @param requests The request channel - sends requests. + * @param replies The reply channel - receives replies. + * @param properties arbitrary String key/value pairs that will be used in the binding. + */ + void bindRequestor(String name, MessageChannel requests, MessageChannel replies, Properties properties); + + /** + * Bind a consumer that handles requests from a requestor and asynchronously sends replies. To unbind, invoke + * unbindProducer() and unbindConsumer(). + * + * @param name The name of the requestor for which this replier will handle requests. + * @param requests The request channel - receives requests. + * @param replies The reply channel - sends replies. + * @param properties arbitrary String key/value pairs that will be used in the binding. + */ + void bindReplier(String name, MessageChannel requests, MessageChannel replies, Properties properties); + + /** + * Create a channel and bind a producer dynamically, creating the infrastructure + * required by the bus technology. + * @param name The name of the "queue:" channel. + * @param properties arbitrary String key/value pairs that will be used in the binding. + * @return The channel. + */ + MessageChannel bindDynamicProducer(String name, Properties properties); + + /** + * Create a channel and bind a producer dynamically, creating the infrastructure + * required by the bus technology to broadcast messages to consumers. + * @param name The name of the "topic:" channel. + * @param properties arbitrary String key/value pairs that will be used in the binding. + * @return The channel. + */ + MessageChannel bindDynamicPubSubProducer(String name, Properties properties); + + /** + * Return true if the bus supports the capability. + * @param capability the capability. + * @return true if the capability is supported. + */ + boolean isCapable(Capability capability); + + public enum Capability { + + /** + * When a bus supports durable subscriptions to a pub/sub channel, the stream + * name will be included in the consumer name. + */ + DURABLE_PUBSUB + + } + +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/MessageBusException.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/MessageBusException.java new file mode 100644 index 000000000..e9ce7f249 --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/MessageBusException.java @@ -0,0 +1,33 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.xd.dirt.integration.bus; + + +/** + * Exception thrown to indicate a message bus error (most + * likely a configuration error). + * + * @author Gary Russell + */ +@SuppressWarnings("serial") +public class MessageBusException extends RuntimeException { + + public MessageBusException(String message, Throwable cause) { + super(message, cause); + } + +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/MessageBusSupport.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/MessageBusSupport.java new file mode 100644 index 000000000..a12be4530 --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/MessageBusSupport.java @@ -0,0 +1,1133 @@ +/* + * Copyright 2013-2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on + * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the + * specific language governing permissions and limitations under the License. + */ + +package org.springframework.xd.dirt.integration.bus; + +import static org.springframework.util.MimeTypeUtils.ALL; +import static org.springframework.util.MimeTypeUtils.APPLICATION_OCTET_STREAM; +import static org.springframework.util.MimeTypeUtils.APPLICATION_OCTET_STREAM_VALUE; +import static org.springframework.util.MimeTypeUtils.TEXT_PLAIN; +import static org.springframework.util.MimeTypeUtils.TEXT_PLAIN_VALUE; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; +import java.io.UnsupportedEncodingException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashSet; +import java.util.Iterator; +import java.util.LinkedList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import java.util.Properties; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.springframework.beans.BeansException; +import org.springframework.beans.factory.BeanFactory; +import org.springframework.beans.factory.InitializingBean; +import org.springframework.beans.factory.config.ConfigurableListableBeanFactory; +import org.springframework.beans.factory.support.DefaultListableBeanFactory; +import org.springframework.context.ApplicationContext; +import org.springframework.context.ApplicationContextAware; +import org.springframework.context.Lifecycle; +import org.springframework.context.support.AbstractApplicationContext; +import org.springframework.expression.EvaluationContext; +import org.springframework.expression.Expression; +import org.springframework.integration.channel.DirectChannel; +import org.springframework.integration.context.IntegrationContextUtils; +import org.springframework.integration.endpoint.EventDrivenConsumer; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageChannel; +import org.springframework.messaging.MessageHandler; +import org.springframework.messaging.MessageHeaders; +import org.springframework.messaging.MessagingException; +import org.springframework.messaging.SubscribableChannel; +import org.springframework.retry.backoff.ExponentialBackOffPolicy; +import org.springframework.retry.policy.SimpleRetryPolicy; +import org.springframework.retry.support.RetryTemplate; +import org.springframework.util.AlternativeJdkIdGenerator; +import org.springframework.util.Assert; +import org.springframework.util.ClassUtils; +import org.springframework.util.IdGenerator; +import org.springframework.util.MimeType; +import org.springframework.util.StringUtils; +import org.springframework.xd.dirt.integration.bus.serializer.MultiTypeCodec; +import org.springframework.xd.dirt.integration.bus.serializer.SerializationException; + +/** + * @author David Turanski + * @author Gary Russell + * @author Ilayaperumal Gopinathan + */ +public abstract class MessageBusSupport + implements MessageBus, ApplicationContextAware, InitializingBean { + + protected static final String P2P_NAMED_CHANNEL_TYPE_PREFIX = "queue:"; + + protected static final String PUBSUB_NAMED_CHANNEL_TYPE_PREFIX = "topic:"; + + protected static final String JOB_CHANNEL_TYPE_PREFIX = "job:"; + + protected static final String PARTITION_HEADER = "partition"; + + protected final Logger logger = LoggerFactory.getLogger(getClass()); + + private volatile AbstractApplicationContext applicationContext; + + private volatile MultiTypeCodec codec; + + private final StringConvertingContentTypeResolver contentTypeResolver = new StringConvertingContentTypeResolver(); + + private final ThreadLocal revertingDirectBinding = new ThreadLocal(); + + protected static final List MEDIATYPES_MEDIATYPE_ALL = Collections.singletonList(ALL); + + private static final int DEFAULT_BACKOFF_INITIAL_INTERVAL = 1000; + + private static final int DEFAULT_BACKOFF_MAX_INTERVAL = 10000; + + private static final double DEFAULT_BACKOFF_MULTIPLIER = 2.0; + + private static final int DEFAULT_CONCURRENCY = 1; + + private static final int DEFAULT_MAX_ATTEMPTS = 3; + + private static final int DEFAULT_BATCH_SIZE = 50; + + private static final int DEFAULT_BATCH_BUFFER_LIMIT = 10000; + + private static final int DEFAULT_BATCH_TIMEOUT = 0; + + /** + * The set of properties every bus implementation must support (or at least tolerate). + */ + + protected static final Set CONSUMER_STANDARD_PROPERTIES = new SetBuilder() + .add(BusProperties.COUNT) + .add(BusProperties.SEQUENCE) + .build(); + + protected static final Set PRODUCER_STANDARD_PROPERTIES = new HashSet(Arrays.asList( + BusProperties.NEXT_MODULE_COUNT, + BusProperties.NEXT_MODULE_CONCURRENCY + )); + + + protected static final Set CONSUMER_RETRY_PROPERTIES = new HashSet(Arrays.asList(new String[] { + BusProperties.BACK_OFF_INITIAL_INTERVAL, + BusProperties.BACK_OFF_MAX_INTERVAL, + BusProperties.BACK_OFF_MULTIPLIER, + BusProperties.MAX_ATTEMPTS + })); + + protected static final Set PRODUCER_PARTITIONING_PROPERTIES = new HashSet( + Arrays.asList(new String[] { + BusProperties.PARTITION_KEY_EXPRESSION, + BusProperties.PARTITION_KEY_EXTRACTOR_CLASS, + BusProperties.PARTITION_SELECTOR_CLASS, + BusProperties.PARTITION_SELECTOR_EXPRESSION, + })); + + protected static final Set PRODUCER_BATCHING_BASIC_PROPERTIES = new HashSet( + Arrays.asList(new String[] { + BusProperties.BATCHING_ENABLED, + BusProperties.BATCH_SIZE, + BusProperties.BATCH_TIMEOUT, + })); + + protected static final Set PRODUCER_BATCHING_ADVANCED_PROPERTIES = new HashSet( + Arrays.asList(new String[] { + BusProperties.BATCH_BUFFER_LIMIT, + })); + + private final List bindings = Collections.synchronizedList(new ArrayList()); + + private final IdGenerator idGenerator = new AlternativeJdkIdGenerator(); + + protected volatile EvaluationContext evaluationContext; + + private volatile PartitionSelectorStrategy partitionSelector = new DefaultPartitionSelector(); + + /** + * Used in the canonical case, when the binding does not involve an alias name. + */ + protected final SharedChannelProvider directChannelProvider = new + SharedChannelProvider( + DirectChannel.class) { + + @Override + protected DirectChannel createSharedChannel(String name) { + return new DirectChannel(); + } + }; + + protected volatile long defaultBackOffInitialInterval = DEFAULT_BACKOFF_INITIAL_INTERVAL; + + protected volatile long defaultBackOffMaxInterval = DEFAULT_BACKOFF_MAX_INTERVAL; + + protected volatile double defaultBackOffMultiplier = DEFAULT_BACKOFF_MULTIPLIER; + + protected volatile int defaultConcurrency = DEFAULT_CONCURRENCY; + + protected volatile int defaultMaxAttempts = DEFAULT_MAX_ATTEMPTS; + + // properties for bus implementations that support batching + + protected volatile boolean defaultBatchingEnabled = false; + + protected volatile int defaultBatchSize = DEFAULT_BATCH_SIZE; + + protected volatile int defaultBatchBufferLimit = DEFAULT_BATCH_BUFFER_LIMIT; + + protected volatile long defaultBatchTimeout = DEFAULT_BATCH_TIMEOUT; + + // compression + + protected volatile boolean defaultCompress = false; + + protected volatile boolean defaultDurableSubscription = false; + + // Payload type cache + private volatile Map> payloadTypeCache = new ConcurrentHashMap<>(); + + /** + * For bus implementations that support a prefix, apply the prefix to the name. + * @param prefix the prefix. + * @param name the name. + */ + public static String applyPrefix(String prefix, String name) { + return prefix + name; + } + + /** + * For bus implementations that include a pub/sub component in identifiers, construct the name. + * @param name the name. + */ + public static String applyPubSub(String name) { + return "topic." + name; + } + + /** + * Build the requests entity name. + * @param name the name. + * @return the request entity name. + */ + public static String applyRequests(String name) { + return name + ".requests"; + } + + /** + * For bus implementations that support dead lettering, construct the name of the dead letter entity for the + * underlying pipe name. + * @param name the name. + */ + public static String constructDLQName(String name) { + return name + ".dlq"; + } + + @Override + public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { + Assert.isInstanceOf(AbstractApplicationContext.class, applicationContext); + this.applicationContext = (AbstractApplicationContext) applicationContext; + } + + protected AbstractApplicationContext getApplicationContext() { + return this.applicationContext; + } + + protected ConfigurableListableBeanFactory getBeanFactory() { + return this.applicationContext.getBeanFactory(); + } + + public void setCodec(MultiTypeCodec codec) { + this.codec = codec; + } + + protected IdGenerator getIdGenerator() { + return idGenerator; + } + + public void setIntegrationEvaluationContext(EvaluationContext evaluationContext) { + this.evaluationContext = evaluationContext; + } + + /** + * Set the partition strategy to be used by this bus if no partitionExpression is provided for a module. + * @param partitionSelector The selector. + */ + public void setPartitionSelector(PartitionSelectorStrategy partitionSelector) { + this.partitionSelector = partitionSelector; + } + + /** + * Set the default retry back off initial interval for this bus; can be overridden with consumer + * 'backOffInitialInterval' property. + * @param defaultBackOffInitialInterval + */ + public void setDefaultBackOffInitialInterval(long defaultBackOffInitialInterval) { + this.defaultBackOffInitialInterval = defaultBackOffInitialInterval; + } + + /** + * Set the default retry back off multiplier for this bus; can be overridden with consumer 'backOffMultiplier' + * property. + * @param defaultBackOffMultiplier + */ + public void setDefaultBackOffMultiplier(double defaultBackOffMultiplier) { + this.defaultBackOffMultiplier = defaultBackOffMultiplier; + } + + /** + * Set the default retry back off max interval for this bus; can be overridden with consumer 'backOffMaxInterval' + * property. + * @param defaultBackOffMaxInterval + */ + public void setDefaultBackOffMaxInterval(long defaultBackOffMaxInterval) { + this.defaultBackOffMaxInterval = defaultBackOffMaxInterval; + } + + /** + * Set the default concurrency for this bus; can be overridden with consumer 'concurrency' property. + * @param defaultConcurrency + */ + public void setDefaultConcurrency(int defaultConcurrency) { + this.defaultConcurrency = defaultConcurrency; + } + + /** + * The default maximum delivery attempts for this bus. Can be overridden by consumer property 'maxAttempts' if + * supported. Values less than 2 disable retry and one delivery attempt is made. + * @param defaultMaxAttempts The default maximum attempts. + */ + public void setDefaultMaxAttempts(int defaultMaxAttempts) { + this.defaultMaxAttempts = defaultMaxAttempts; + } + + /** + * Set whether this bus batches message sends by default. Only applies to bus implementations that support + * batching. + * @param defaultBatchingEnabled the defaultBatchingEnabled to set. + */ + public void setDefaultBatchingEnabled(boolean defaultBatchingEnabled) { + this.defaultBatchingEnabled = defaultBatchingEnabled; + } + + /** + * Set the default batch size; only applies when batching is enabled and the bus supports batching. + * @param defaultBatchSize the defaultBatchSize to set. + */ + public void setDefaultBatchSize(int defaultBatchSize) { + this.defaultBatchSize = defaultBatchSize; + } + + /** + * Set the default batch buffer limit - used to send a batch early if its size exceeds this. Only applies if + * batching is enabled and the bus supports this property. + * @param defaultBatchBufferLimit the defaultBatchBufferLimit to set. + */ + public void setDefaultBatchBufferLimit(int defaultBatchBufferLimit) { + this.defaultBatchBufferLimit = defaultBatchBufferLimit; + } + + /** + * Set the default batch timeout - used to send a batch if no messages arrive during this time. Only applies if + * batching is enabled and the bus supports this property. + * @param defaultBatchTimeout the defaultBatchTimeout to set. + */ + public void setDefaultBatchTimeout(long defaultBatchTimeout) { + this.defaultBatchTimeout = defaultBatchTimeout; + } + + /** + * Set whether compression will be used by producers, by default. + * @param defaultCompress 'true' to use compression. + */ + public void setDefaultCompress(boolean defaultCompress) { + this.defaultCompress = defaultCompress; + } + + /** + * Set whether subscriptions to taps/topics are durable. + * @param defaultDurableSubscription true for durable (default false). + */ + public void setDefaultDurableSubscription(boolean defaultDurableSubscription) { + this.defaultDurableSubscription = defaultDurableSubscription; + } + + @Override + public void afterPropertiesSet() throws Exception { + Assert.notNull(applicationContext, "The 'applicationContext' property cannot be null"); + onInit(); + if (this.evaluationContext == null) { + this.evaluationContext = IntegrationContextUtils.getEvaluationContext(getBeanFactory()); + } + } + + protected void onInit() { + } + + /** + * Dynamically create a producer for the named channel. + * @param name The name. + * @param properties The properties. + * @return The channel. + */ + @Override + public MessageChannel bindDynamicProducer(String name, Properties properties) { + return doBindDynamicProducer(name, name, properties); + } + + /** + * Create a producer for the named channel and bind it to the bus. Synchronized to avoid creating multiple + * instances. + * @param name The name. + * @param channelName The name of the channel to be created, and registered as bean. + * @param properties The properties. + * @return The channel. + */ + protected synchronized MessageChannel doBindDynamicProducer(String name, String channelName, + Properties properties) { + MessageChannel channel = this.directChannelProvider.lookupSharedChannel(channelName); + if (channel == null) { + try { + channel = this.directChannelProvider.createAndRegisterChannel(channelName); + bindProducer(name, channel, properties); + } + catch (RuntimeException e) { + destroyCreatedChannel(channelName, channel); + throw new MessageBusException( + "Failed to bind dynamic channel '" + name + "' with properties " + properties, e); + } + } + return channel; + } + + /** + * Dynamically create a producer for the named channel. Note: even though it's pub/sub, we still use a direct + * channel. It will be bridged to a pub/sub channel in the local bus and bound to an appropriate element for other + * buses. + * @param name The name. + * @param properties The properties. + * @return The channel. + */ + @Override + public MessageChannel bindDynamicPubSubProducer(String name, Properties properties) { + return doBindDynamicPubSubProducer(name, name, properties); + } + + /** + * Create a producer for the named channel and bind it to the bus. Synchronized to avoid creating multiple + * instances. + * @param name The name. + * @param channelName The name of the channel to be created, and registered as bean. + * @param properties The properties. + * @return The channel. + */ + protected synchronized MessageChannel doBindDynamicPubSubProducer(String name, String channelName, + Properties properties) { + MessageChannel channel = this.directChannelProvider.lookupSharedChannel(channelName); + if (channel == null) { + try { + channel = this.directChannelProvider.createAndRegisterChannel(channelName); + bindPubSubProducer(name, channel, properties); + } + catch (RuntimeException e) { + destroyCreatedChannel(channelName, channel); + throw new MessageBusException( + "Failed to bind dynamic channel '" + name + "' with properties " + properties, e); + } + } + return channel; + } + + private void destroyCreatedChannel(String name, MessageChannel channel) { + BeanFactory beanFactory = this.applicationContext.getBeanFactory(); + if (beanFactory.containsBean(name)) { + if (beanFactory instanceof DefaultListableBeanFactory) { + ((DefaultListableBeanFactory) beanFactory).destroySingleton(name); + } + } + } + + @Override + public void unbindConsumers(String name) { + deleteBindings("inbound." + name); + } + + @Override + public void unbindProducers(String name) { + deleteBindings("outbound." + name); + } + + @Override + public void unbindConsumer(String name, MessageChannel channel) { + deleteBinding("inbound." + name, channel); + } + + @Override + public void unbindProducer(String name, MessageChannel channel) { + deleteBinding("outbound." + name, channel); + } + + @Override + public boolean isCapable(Capability capability) { + return false; + } + + protected void addBinding(Binding binding) { + this.bindings.add(binding); + } + + protected void deleteBindings(String name) { + Assert.hasText(name, "a valid name is required to remove bindings"); + List bindingsToRemove = new ArrayList(); + synchronized (this.bindings) { + Iterator iterator = this.bindings.iterator(); + while (iterator.hasNext()) { + Binding binding = iterator.next(); + if (binding.getEndpoint().getComponentName().equals(name)) { + bindingsToRemove.add(binding); + } + } + for (Binding binding : bindingsToRemove) { + doDeleteBinding(binding); + } + } + } + + protected void deleteBinding(String name, MessageChannel channel) { + Assert.hasText(name, "a valid name is required to remove a binding"); + Assert.notNull(channel, "a valid channel is required to remove a binding"); + Binding bindingToRemove = null; + synchronized (this.bindings) { + Iterator iterator = this.bindings.iterator(); + while (iterator.hasNext()) { + Binding binding = iterator.next(); + if (binding.getChannel().equals(channel) && + binding.getEndpoint().getComponentName().equals(name)) { + bindingToRemove = binding; + break; + } + } + if (bindingToRemove != null) { + doDeleteBinding(bindingToRemove); + } + } + + } + + private void doDeleteBinding(Binding binding) { + if (Binding.CONSUMER.equals(binding.getType())) { + /* + * Revert the direct binding before stopping the consumer; the module + * outputChannel will temporarily have 2 subscribers. + */ + revertDirectBindingIfNecessary(binding); + } + binding.stop(); + this.bindings.remove(binding); + } + + protected void stopBindings() { + for (Lifecycle bean : this.bindings) { + try { + bean.stop(); + } + catch (Exception e) { + if (logger.isWarnEnabled()) { + logger.warn("failed to stop adapter", e); + } + } + } + } + + protected final MessageValues serializePayloadIfNecessary(Message message) { + Object originalPayload = message.getPayload(); + Object originalContentType = message.getHeaders().get(MessageHeaders.CONTENT_TYPE); + + //Pass content type as String since some transport adapters will exclude CONTENT_TYPE Header otherwise + Object contentType = JavaClassMimeTypeConversion.mimeTypeFromObject(originalPayload).toString(); + Object payload = serializePayloadIfNecessary(originalPayload); + MessageValues messageValues = new MessageValues(message); + messageValues.setPayload(payload); + messageValues.put(MessageHeaders.CONTENT_TYPE, contentType); + if (originalContentType != null) { + messageValues.put(XdHeaders.XD_ORIGINAL_CONTENT_TYPE, originalContentType); + } + return messageValues; + } + + private byte[] serializePayloadIfNecessary(Object originalPayload) { + if (originalPayload instanceof byte[]) { + return (byte[]) originalPayload; + } + else { + ByteArrayOutputStream bos = new ByteArrayOutputStream(); + try { + if (originalPayload instanceof String) { + return ((String) originalPayload).getBytes("UTF-8"); + } + this.codec.serialize(originalPayload, bos); + return bos.toByteArray(); + } + catch (IOException e) { + throw new SerializationException("unable to serialize payload [" + + originalPayload.getClass().getName() + "]", e); + } + } + } + + protected final MessageValues deserializePayloadIfNecessary(Message message) { + return deserializePayloadIfNecessary(new MessageValues(message)); + } + + protected final MessageValues deserializePayloadIfNecessary(MessageValues message) { + MessageValues messageToSend = message; + Object originalPayload = message.getPayload(); + MimeType contentType = contentTypeResolver.resolve(messageToSend); + Object payload = deserializePayload(originalPayload, contentType); + if (payload != null) { + messageToSend.setPayload(payload); + + Object originalContentType = messageToSend.get(XdHeaders.XD_ORIGINAL_CONTENT_TYPE); + messageToSend.put(MessageHeaders.CONTENT_TYPE, originalContentType); + messageToSend.put(XdHeaders.XD_ORIGINAL_CONTENT_TYPE, null); + } + return messageToSend; + } + + private Object deserializePayload(Object payload, MimeType contentType) { + if (payload instanceof byte[]) { + if (contentType == null || APPLICATION_OCTET_STREAM.equals(contentType)) { + return payload; + } + else { + return deserializePayload((byte[]) payload, contentType); + } + } + return payload; + } + + private Object deserializePayload(byte[] bytes, MimeType contentType) { + if (TEXT_PLAIN.equals(contentType)) { + try { + return new String(bytes, "UTF-8"); + } + catch (UnsupportedEncodingException e) { + throw new SerializationException("unable to deserialize [java.lang.String]. Encoding not supported.", e); + } + } + else { + String className = JavaClassMimeTypeConversion.classNameFromMimeType(contentType); + try { + // Cache types to avoid unnecessary ClassUtils.forName calls. + Class targetType = payloadTypeCache.get(className); + if (targetType == null) { + targetType = ClassUtils.forName(className, null); + payloadTypeCache.put(className, targetType); + } + return codec.deserialize(bytes, targetType); + } catch (ClassNotFoundException e) { + throw new SerializationException("unable to deserialize [" + className + "]. Class not found.", e);//NOSONAR + } catch (IOException e) { + throw new SerializationException("unable to deserialize [" + className + "]", e); + } + } + } + + /** + * Determine the partition to which to send this message. If a partition key extractor class is provided, it is + * invoked to determine the key. Otherwise, the partition key expression is evaluated to obtain the key value. If a + * partition selector class is provided, it will be invoked to determine the partition. Otherwise, if the partition + * expression is not null, it is evaluated against the key and is expected to return an integer to which the modulo + * function will be applied, using the partitionCount as the divisor. If no partition expression is provided, the + * key will be passed to the bus partition strategy along with the partitionCount. The default partition strategy + * uses {@code key.hashCode()}, and the result will be the mod of that value. + * @param message the message. + * @param meta the partitioning metadata. + * @return the partition. + */ + protected int determinePartition(Message message, PartitioningMetadata meta) { + Object key = null; + if (StringUtils.hasText(meta.partitionKeyExtractorClass)) { + key = invokeExtractor(meta.partitionKeyExtractorClass, message); + } + else if (meta.partitionKeyExpression != null) { + key = meta.partitionKeyExpression.getValue(this.evaluationContext, message); + } + Assert.notNull(key, "Partition key cannot be null"); + int partition; + if (StringUtils.hasText(meta.partitionSelectorClass)) { + partition = invokePartitionSelector(meta.partitionSelectorClass, key, meta.partitionCount); + } + else if (meta.partitionSelectorExpression != null) { + partition = meta.partitionSelectorExpression.getValue(this.evaluationContext, key, Integer.class); + } + else { + partition = this.partitionSelector.selectPartition(key, meta.partitionCount); + } + partition = partition % meta.partitionCount; + if (partition < 0) { // protection in case a user selector returns a negative. + partition = Math.abs(partition); + } + return partition; + } + + private Object invokeExtractor(String partitionKeyExtractorClassName, Message message) { + if (this.applicationContext.containsBean(partitionKeyExtractorClassName)) { + return this.applicationContext.getBean(partitionKeyExtractorClassName, PartitionKeyExtractorStrategy.class) + .extractKey(message); + } + Class clazz; + try { + clazz = ClassUtils.forName(partitionKeyExtractorClassName, this.applicationContext.getClassLoader()); + } + catch (Exception e) { + logger.error("Failed to load key extractor", e); + throw new MessageBusException("Failed to load key extractor: " + partitionKeyExtractorClassName, e); + } + try { + Object extractor = clazz.newInstance(); + Assert.isInstanceOf(PartitionKeyExtractorStrategy.class, extractor); + this.applicationContext.getBeanFactory().registerSingleton(partitionKeyExtractorClassName, extractor); + this.applicationContext.getBeanFactory().initializeBean(extractor, partitionKeyExtractorClassName); + return ((PartitionKeyExtractorStrategy) extractor).extractKey(message); + } + catch (Exception e) { + logger.error("Failed to instantiate key extractor", e); + throw new MessageBusException("Failed to instantiate key extractor: " + partitionKeyExtractorClassName, e); + } + } + + private int invokePartitionSelector(String partitionSelectorClassName, Object key, int partitionCount) { + if (this.applicationContext.containsBean(partitionSelectorClassName)) { + return this.applicationContext.getBean(partitionSelectorClassName, PartitionSelectorStrategy.class) + .selectPartition(key, partitionCount); + } + Class clazz; + try { + clazz = ClassUtils.forName(partitionSelectorClassName, this.applicationContext.getClassLoader()); + } + catch (Exception e) { + logger.error("Failed to load partition selector", e); + throw new MessageBusException("Failed to load partition selector: " + partitionSelectorClassName, e); + } + try { + Object extractor = clazz.newInstance(); + Assert.isInstanceOf(PartitionKeyExtractorStrategy.class, extractor); + this.applicationContext.getBeanFactory().registerSingleton(partitionSelectorClassName, extractor); + this.applicationContext.getBeanFactory().initializeBean(extractor, partitionSelectorClassName); + return ((PartitionSelectorStrategy) extractor).selectPartition(key, partitionCount); + } + catch (Exception e) { + logger.error("Failed to instantiate partition selector", e); + throw new MessageBusException("Failed to instantiate partition selector: " + partitionSelectorClassName, + e); + } + } + + /** + * Validate the provided deployment properties for the consumer against those supported by this bus implementation. + * The consumer is that part of the bus that consumes messages from the underlying infrastructure and sends them to + * the next module. Consumer properties are used to configure the consumer. + * @param name The name. + * @param properties The properties. + * @param supported The supported properties. + */ + protected void validateConsumerProperties(String name, Properties properties, Set supported) { + if (properties != null) { + validateProperties(name, properties, supported, "consumer"); + } + } + + /** + * Validate the provided deployment properties for the producer against those supported by this bus implementation. + * When a module sends a message to the bus, the producer uses these properties while sending it to the underlying + * infrastructure. + * @param name The name. + * @param properties The properties. + * @param supported The supported properties. + */ + protected void validateProducerProperties(String name, Properties properties, Set supported) { + if (properties != null) { + validateProperties(name, properties, supported, "producer"); + } + } + + private void validateProperties(String name, Properties properties, Set supported, String type) { + StringBuilder builder = new StringBuilder(); + int errors = 0; + for (Entry entry : properties.entrySet()) { + if (!supported.contains(entry.getKey())) { + builder.append(entry.getKey()).append(","); + errors++; + } + } + if (errors > 0) { + throw new IllegalArgumentException(getClass().getSimpleName() + " does not support " + + type + + " propert" + + (errors == 1 ? "y: " : "ies: ") + + builder.substring(0, builder.length() - 1) + + " for " + name + "."); + } + } + + protected String buildPartitionRoutingExpression(String expressionRoot) { + return "'" + expressionRoot + "-' + headers['" + PARTITION_HEADER + "']"; + } + + /** + * Create and configure a retry template if the consumer 'maxAttempts' property is set. + * @param properties The properties. + * @return The retry template, or null if retry is not enabled. + */ + protected RetryTemplate buildRetryTemplateIfRetryEnabled(AbstractBusPropertiesAccessor properties) { + int maxAttempts = properties.getMaxAttempts(this.defaultMaxAttempts); + if (maxAttempts > 1) { + RetryTemplate template = new RetryTemplate(); + SimpleRetryPolicy retryPolicy = new SimpleRetryPolicy(); + retryPolicy.setMaxAttempts(maxAttempts); + ExponentialBackOffPolicy backOffPolicy = new ExponentialBackOffPolicy(); + backOffPolicy.setInitialInterval(properties.getBackOffInitialInterval(this.defaultBackOffInitialInterval)); + backOffPolicy.setMultiplier(properties.getBackOffMultiplier(this.defaultBackOffMultiplier)); + backOffPolicy.setMaxInterval(properties.getBackOffMaxInterval(this.defaultBackOffMaxInterval)); + template.setRetryPolicy(retryPolicy); + template.setBackOffPolicy(backOffPolicy); + return template; + } + else { + return null; + } + } + + protected boolean isNamedChannel(String name) { + return name.startsWith(PUBSUB_NAMED_CHANNEL_TYPE_PREFIX) || name.startsWith(P2P_NAMED_CHANNEL_TYPE_PREFIX) + || name.startsWith(JOB_CHANNEL_TYPE_PREFIX); + } + + /** + * Attempt to create a direct binding (avoiding the bus) if the consumer is local. Named channel producers are not + * bound directly. + * @param name The name. + * @param moduleOutputChannel The channel to bind. + * @param properties The producer properties. + * @return true if the producer is bound. + */ + protected boolean bindNewProducerDirectlyIfPossible(String name, SubscribableChannel moduleOutputChannel, + AbstractBusPropertiesAccessor properties) { + if (!properties.isDirectBindingAllowed()) { + return false; + } + else if (isNamedChannel(name)) { + return false; + } + else if (this.revertingDirectBinding.get() != null) { + // we're in the process of unbinding a direct binding + this.revertingDirectBinding.remove(); + return false; + } + else { + Binding consumerBinding = null; + synchronized (this.bindings) { + for (Binding binding : this.bindings) { + if (binding.getName().equals(name) && Binding.CONSUMER.equals(binding.getType())) { + consumerBinding = binding; + break; + } + } + } + if (consumerBinding == null) { + return false; + } + else { + bindProducerDirectly(name, moduleOutputChannel, consumerBinding.getChannel(), properties); + return true; + } + } + } + + private void bindProducerDirectly(String name, SubscribableChannel producerChannel, + MessageChannel consumerChannel, AbstractBusPropertiesAccessor properties) { + DirectHandler handler = new DirectHandler(consumerChannel); + EventDrivenConsumer consumer = new EventDrivenConsumer(producerChannel, handler); + consumer.setBeanFactory(getBeanFactory()); + consumer.setBeanName("outbound." + name); + consumer.afterPropertiesSet(); + Binding binding = Binding.forDirectProducer(name, producerChannel, consumer, properties); + addBinding(binding); + binding.start(); + if (logger.isInfoEnabled()) { + logger.info("Producer bound directly: " + binding); + } + } + + /** + * Attempt to bind a producer directly (avoiding the bus) if there is already a local producer. PubSub producers + * cannot be bound directly. Create the direct binding, then unbind the existing bus producer. + * @param name The name. + * @param consumerChannel The channel to bind the producer to. + */ + protected void bindExistingProducerDirectlyIfPossible(String name, MessageChannel consumerChannel) { + if (!isNamedChannel(name)) { + Binding producerBinding = null; + synchronized (this.bindings) { + for (Binding binding : this.bindings) { + if (binding.getName().equals(name) && Binding.PRODUCER.equals(binding.getType())) { + producerBinding = binding; + break; + } + } + if (producerBinding != null && producerBinding.getChannel() instanceof SubscribableChannel) { + AbstractBusPropertiesAccessor properties = producerBinding.getPropertiesAccessor(); + if (properties.isDirectBindingAllowed()) { + bindProducerDirectly(name, (SubscribableChannel) producerBinding.getChannel(), consumerChannel, + properties); + producerBinding.stop(); + this.bindings.remove(producerBinding); + } + } + } + } + } + + private void revertDirectBindingIfNecessary(Binding binding) { + try { + synchronized (this.bindings) { // Not necessary, called while synchronized, but just in case... + Binding directBinding = null; + Iterator iterator = this.bindings.iterator(); + while (iterator.hasNext()) { + Binding producer = iterator.next(); + if (Binding.DIRECT.equals(producer.getType()) && binding.getName().equals(producer.getName())) { + this.revertingDirectBinding.set(Boolean.TRUE); + bindProducer(producer.getName(), producer.getChannel(), + producer.getPropertiesAccessor().getProperties()); + directBinding = producer; + break; + } + } + if (directBinding != null) { + directBinding.stop(); + this.bindings.remove(directBinding); + if (logger.isInfoEnabled()) { + logger.info("direct binding reverted: " + directBinding); + } + } + } + } + catch (Exception e) { + logger.error("Could not revert direct binding: " + binding, e); + } + } + + /** + * Default partition strategy; only works on keys with "real" hash codes, such as String. Caller now always applies + * modulo so no need to do so here. + */ + private class DefaultPartitionSelector implements PartitionSelectorStrategy { + + @Override + public int selectPartition(Object key, int partitionCount) { + int hashCode = key.hashCode(); + if (hashCode == Integer.MIN_VALUE) { + hashCode = 0; + } + return Math.abs(hashCode); + } + + } + + protected static class PartitioningMetadata { + + private final String partitionKeyExtractorClass; + + private final Expression partitionKeyExpression; + + private final String partitionSelectorClass; + + private final Expression partitionSelectorExpression; + + private final int partitionCount; + + public PartitioningMetadata(AbstractBusPropertiesAccessor properties, int partitionCount) { + this.partitionCount = partitionCount; + this.partitionKeyExtractorClass = properties.getPartitionKeyExtractorClass(); + this.partitionKeyExpression = properties.getPartitionKeyExpression(); + this.partitionSelectorClass = properties.getPartitionSelectorClass(); + this.partitionSelectorExpression = properties.getPartitionSelectorExpression(); + } + + public boolean isPartitionedModule() { + return StringUtils.hasText(this.partitionKeyExtractorClass) || this.partitionKeyExpression != null; + } + + public int getPartitionCount() { + return partitionCount; + } + } + + /** + * Looks up or optionally creates a new channel to use. + * @author Eric Bottard + */ + protected abstract class SharedChannelProvider { + + private final Class requiredType; + + protected SharedChannelProvider(Class clazz) { + this.requiredType = clazz; + } + + public synchronized final T lookupOrCreateSharedChannel(String name) { + T channel = lookupSharedChannel(name); + if (channel == null) { + channel = createAndRegisterChannel(name); + } + return channel; + } + + @SuppressWarnings("unchecked") + public T createAndRegisterChannel(String name) { + T channel = createSharedChannel(name); + ConfigurableListableBeanFactory beanFactory = applicationContext.getBeanFactory(); + beanFactory.registerSingleton(name, channel); + channel = (T) beanFactory.initializeBean(channel, name); + if (logger.isDebugEnabled()) { + logger.debug("Registered channel:" + name); + } + return channel; + } + + protected abstract T createSharedChannel(String name); + + public T lookupSharedChannel(String name) { + T channel = null; + if (applicationContext.containsBean(name)) { + try { + channel = applicationContext.getBean(name, requiredType); + } + catch (Exception e) { + throw new IllegalArgumentException("bean '" + name + + "' is already registered but does not match the required type"); + } + } + return channel; + } + } + + /** + * Handles representing any java class as a {@link MimeType}. + * @author David Turanski + * @see + */ + abstract static class JavaClassMimeTypeConversion { + + public static final MimeType APPLICATION_OCTET_STREAM_MIME_TYPE = MimeType.valueOf(APPLICATION_OCTET_STREAM_VALUE); + + public static final MimeType TEXT_PLAIN_MIME_TYPE = MimeType.valueOf(TEXT_PLAIN_VALUE); + + private static ConcurrentMap mimeTypesCache = new ConcurrentHashMap<>(); + + static MimeType mimeTypeFromObject(Object obj) { + Assert.notNull(obj, "object cannot be null."); + if (obj instanceof byte[]) { + return APPLICATION_OCTET_STREAM_MIME_TYPE; + } + if (obj instanceof String) { + return TEXT_PLAIN_MIME_TYPE; + } + String className = obj.getClass().getName(); + MimeType mimeType = mimeTypesCache.get(className); + if (mimeType == null) { + String modifiedClassName = className; + if (obj.getClass().isArray()) { + // Need to remove trailing ';' for an object array, e.g. "[Ljava.lang.String;" or multi-dimensional + // "[[[Ljava.lang.String;" + if (modifiedClassName.endsWith(";")) { + modifiedClassName = modifiedClassName.substring(0, modifiedClassName.length() - 1); + } + // Wrap in quotes to handle the illegal '[' character + modifiedClassName = "\"" + modifiedClassName + "\""; + } + mimeType = MimeType.valueOf("application/x-java-object;type=" + modifiedClassName); + mimeTypesCache.put(className, mimeType); + } + return mimeType; + } + + static String classNameFromMimeType(MimeType mimeType) { + Assert.notNull(mimeType, "mimeType cannot be null."); + String className = mimeType.getParameter("type"); + if (className == null) { + return null; + } + //unwrap quotes if any + className = className.replace("\"", ""); + + // restore trailing ';' + if (className.contains("[L")) { + className += ";"; + } + return className; + } + } + + public static class SetBuilder { + + private final Set set = new HashSet(); + + public SetBuilder add(Object o) { + this.set.add(o); + return this; + } + + public SetBuilder addAll(Set set) { + this.set.addAll(set); + return this; + } + + public Set build() { + return this.set; + } + + } + + public static class DirectHandler implements MessageHandler { + + private final MessageChannel outputChannel; + + public DirectHandler(MessageChannel outputChannel) { + this.outputChannel = outputChannel; + } + + @Override + public void handleMessage(Message message) throws MessagingException { + this.outputChannel.send(message); + } + + } + + /** + * Perform manual acknowledgement based on the metadata stored in message bus. + */ + public void doManualAck(LinkedList messageHeaders) { + } + +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/MessageValues.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/MessageValues.java new file mode 100644 index 000000000..959f29691 --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/MessageValues.java @@ -0,0 +1,155 @@ +/* + * Copyright 2015 the original author or authors. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.xd.dirt.integration.bus; + +import java.util.Collection; +import java.util.HashMap; +import java.util.Map; +import java.util.Set; + +import org.springframework.integration.support.MessageBuilder; +import org.springframework.integration.support.MessageBuilderFactory; +import org.springframework.messaging.Message; +import org.springframework.util.Assert; + +/** + * A mutable type for allowing {@link MessageBus} implementations to transform and enrich message content more + * efficiently. + * @author David Turanski + */ +public class MessageValues implements Map { + private Map headers = new HashMap<>(); + + private Object payload; + + /** + * Create an instance from a {@link Message}. + * @param message the message + */ + public MessageValues(Message message) { + this.payload = message.getPayload(); + for (Map.Entry header : message.getHeaders().entrySet()) { + this.headers.put(header.getKey(), header.getValue()); + } + } + + public MessageValues(Object payload, Map headers) { + this.payload = payload; + this.headers.putAll(headers); + } + + /** + * @return the payload + */ + public Object getPayload() { + return payload; + } + + /** + * Convert to a {@link Message} using a {@link org.springframework.integration.support.MessageBuilderFactory}. + * @param messageBuilderFactory the MessageBuilderFactory + * @return the Message + */ + public Message toMessage(MessageBuilderFactory messageBuilderFactory) { + return messageBuilderFactory.withPayload(this.payload).copyHeaders(this.headers).build(); + } + + + /** + * Convert to a {@link Message} using a the default {@link org.springframework.integration.support.MessageBuilder}. + * @return the Message + */ + public Message toMessage() { + return MessageBuilder.withPayload(this.payload).copyHeaders(this.headers).build(); + } + + /** + * Set the payload + * @param payload any non null object. + */ + public void setPayload(Object payload) { + Assert.notNull(payload, "'payload' cannot be null"); + this.payload = payload; + } + + @Override + public int size() { + return headers.size(); + } + + @Override + public boolean isEmpty() { + return headers.isEmpty(); + } + + @Override + public boolean containsKey(Object key) { + return headers.containsKey(key); + } + + @Override + public boolean containsValue(Object value) { + return headers.containsValue(value); + } + + @Override + public Object get(Object key) { + return headers.get(key); + } + + @Override + public Object put(String key, Object value) { + return headers.put(key, value); + } + + @Override + public Object remove(Object key) { + return headers.remove(key); + } + + @Override + public void putAll(Map m) { + headers.putAll(m); + } + + @Override + public void clear() { + headers.clear(); + } + + @Override + public Set keySet() { + return headers.keySet(); + } + + @Override + public Collection values() { + return headers.values(); + } + + @Override + public Set> entrySet() { + return headers.entrySet(); + } + + public void copyHeadersIfAbsent(Map headersToCopy) { + for (Entry headersToCopyEntry : headersToCopy.entrySet()) { + if (!containsKey(headersToCopyEntry.getKey())) { + put(headersToCopyEntry.getKey(), headersToCopyEntry.getValue()); + } + } + } +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/PartitionKeyExtractorStrategy.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/PartitionKeyExtractorStrategy.java new file mode 100644 index 000000000..cecd3af14 --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/PartitionKeyExtractorStrategy.java @@ -0,0 +1,31 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.xd.dirt.integration.bus; + +import org.springframework.messaging.Message; + + +/** + * Strategy for extracting a partition key from a Message. + * + * @author Gary Russell + */ +public interface PartitionKeyExtractorStrategy { + + Object extractKey(Message message); + +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/PartitionSelectorStrategy.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/PartitionSelectorStrategy.java new file mode 100644 index 000000000..c4a0ea2ca --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/PartitionSelectorStrategy.java @@ -0,0 +1,41 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.xd.dirt.integration.bus; + + +/** + * Strategy for determining the partition to which a message should be sent. + * + * @author Gary Russell + */ +public interface PartitionSelectorStrategy { + + /** + * Determine the partition based on a key. The partitionCount is 1 greater + * than the maximum value of a valid partition. Typical implementations + * will return {@code someValue % partitionCount}. The caller will apply + * that same modulo operation (as well as enforcing absolute value) if the + * value exceeds partitionCount - 1. + * + * @param key the key + * @param partitionCount the number of partitions + * + * @return the partition + */ + int selectPartition(Object key, int partitionCount); + +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/RabbitAdminException.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/RabbitAdminException.java new file mode 100644 index 000000000..bbc2798a1 --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/RabbitAdminException.java @@ -0,0 +1,39 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.xd.dirt.integration.bus; + + +import org.springframework.cloud.streams.exception.CloudStreamsRuntimeException; + +/** + * Exceptions thrown while interfacing with the RabbitMQ admin plugin. + * + * @author Gary Russell + * @since 1.2 + */ +@SuppressWarnings("serial") +public class RabbitAdminException extends CloudStreamsRuntimeException { + + public RabbitAdminException(String message, Throwable cause) { + super(message, cause); + } + + public RabbitAdminException(String message) { + super(message); + } + +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/RabbitManagementUtils.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/RabbitManagementUtils.java new file mode 100644 index 000000000..2bcd6f9e8 --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/RabbitManagementUtils.java @@ -0,0 +1,82 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.xd.dirt.integration.bus; + +import java.net.URI; +import java.net.URISyntaxException; +import java.util.Collections; + +import org.apache.http.HttpHost; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.AuthCache; +import org.apache.http.client.HttpClient; +import org.apache.http.client.protocol.HttpClientContext; +import org.apache.http.impl.auth.BasicScheme; +import org.apache.http.impl.client.BasicAuthCache; +import org.apache.http.impl.client.BasicCredentialsProvider; +import org.apache.http.impl.client.HttpClients; +import org.apache.http.protocol.HttpContext; + +import org.springframework.http.HttpMethod; +import org.springframework.http.client.HttpComponentsClientHttpRequestFactory; +import org.springframework.http.converter.HttpMessageConverter; +import org.springframework.http.converter.json.MappingJackson2HttpMessageConverter; +import org.springframework.web.client.RestTemplate; + +/** + * @author Gary Russell + * @since 1.2 + */ +public class RabbitManagementUtils { + + public static RestTemplate buildRestTemplate(String adminUri, String user, String password) { + BasicCredentialsProvider credsProvider = new BasicCredentialsProvider(); + credsProvider.setCredentials( + new AuthScope(AuthScope.ANY_HOST, AuthScope.ANY_PORT), + new UsernamePasswordCredentials(user, password)); + HttpClient httpClient = HttpClients.custom().setDefaultCredentialsProvider(credsProvider).build(); + // Set up pre-emptive basic Auth because the rabbit plugin doesn't currently support challenge/response for PUT + // Create AuthCache instance + AuthCache authCache = new BasicAuthCache(); + // Generate BASIC scheme object and add it to the local; from the apache docs... + // auth cache + BasicScheme basicAuth = new BasicScheme(); + URI uri; + try { + uri = new URI(adminUri); + } + catch (URISyntaxException e) { + throw new RabbitAdminException("Invalid URI", e); + } + authCache.put(new HttpHost(uri.getHost(), uri.getPort(), uri.getScheme()), basicAuth); + // Add AuthCache to the execution context + final HttpClientContext localContext = HttpClientContext.create(); + localContext.setAuthCache(authCache); + RestTemplate restTemplate = new RestTemplate(new HttpComponentsClientHttpRequestFactory(httpClient) { + + @Override + protected HttpContext createHttpContext(HttpMethod httpMethod, URI uri) { + return localContext; + } + + }); + restTemplate.setMessageConverters(Collections.>singletonList( + new MappingJackson2HttpMessageConverter())); + return restTemplate; + } + +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/StringConvertingContentTypeResolver.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/StringConvertingContentTypeResolver.java new file mode 100644 index 000000000..e68f7f5e8 --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/StringConvertingContentTypeResolver.java @@ -0,0 +1,57 @@ +/* + * Copyright 2002-2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.xd.dirt.integration.bus; + +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentMap; + +import org.springframework.messaging.MessageHeaders; +import org.springframework.messaging.converter.DefaultContentTypeResolver; +import org.springframework.util.MimeType; + +/** + * A {@link DefaultContentTypeResolver} that can parse String values. + * + * @author David Turanski + */ +public class StringConvertingContentTypeResolver extends DefaultContentTypeResolver { + + private ConcurrentMap mimeTypeCache = new ConcurrentHashMap<>(); + + @Override + public MimeType resolve(MessageHeaders headers) { + return resolve((Map) headers); + } + + public MimeType resolve(Map headers) { + Object value = headers.get(MessageHeaders.CONTENT_TYPE); + if (value instanceof MimeType) { + return (MimeType) value; + } + else if (value instanceof String) { + MimeType mimeType = mimeTypeCache.get(value); + if (mimeType == null) { + String valueAsString = (String) value; + mimeType = MimeType.valueOf(valueAsString); + mimeTypeCache.put(valueAsString,mimeType); + } + return mimeType; + } + return getDefaultMimeType(); + } +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/XdHeaders.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/XdHeaders.java new file mode 100644 index 000000000..c26f392be --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/main/java/org/springframework/xd/dirt/integration/bus/XdHeaders.java @@ -0,0 +1,62 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.xd.dirt.integration.bus; + +import org.springframework.integration.IntegrationMessageHeaderAccessor; +import org.springframework.messaging.MessageHeaders; + + +/** + * Spring Integration message headers for XD. + * + * @author Gary Russell + */ +public final class XdHeaders { + + public static final String XD_REPLY_CHANNEL = "xdReplyChannel"; + + public static final String XD_HISTORY = "xdHistory"; + + /* + * no xd prefix for backwards compatibility + */ + public static final String XD_ORIGINAL_CONTENT_TYPE = "originalContentType"; + + /* + * no xd prefix for backwards compatibility + */ + public static final String REPLY_TO = "replyTo"; + + /** + * The headers that will be propagated, by default, by message bus implementations + * that have no inherent header support (by embedding the headers in the payload). + */ + public static final String[] STANDARD_HEADERS = new String[] { + IntegrationMessageHeaderAccessor.CORRELATION_ID, + IntegrationMessageHeaderAccessor.SEQUENCE_SIZE, + IntegrationMessageHeaderAccessor.SEQUENCE_NUMBER, + XD_REPLY_CHANNEL, + MessageHeaders.CONTENT_TYPE, + XD_ORIGINAL_CONTENT_TYPE, + REPLY_TO, + XD_HISTORY + }; + + private XdHeaders() { + } + +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/test/java/org/springframework/xd/dirt/integration/bus/MessageConverterTests.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/test/java/org/springframework/xd/dirt/integration/bus/MessageConverterTests.java new file mode 100644 index 000000000..0fbaaa876 --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-spi/src/test/java/org/springframework/xd/dirt/integration/bus/MessageConverterTests.java @@ -0,0 +1,96 @@ +/* + * Copyright 2002-2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.xd.dirt.integration.bus; + +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.startsWith; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertThat; + +import org.junit.Assert; +import org.junit.Test; + +import org.springframework.integration.support.MessageBuilder; +import org.springframework.messaging.Message; +import org.springframework.messaging.support.GenericMessage; + +/** + * @author Gary Russell + * @since 1.0 + * + */ +public class MessageConverterTests { + + @Test + public void testHeaderEmbedding() throws Exception { + EmbeddedHeadersMessageConverter converter = new EmbeddedHeadersMessageConverter(); + Message message = MessageBuilder.withPayload("Hello".getBytes()) + .setHeader("foo", "bar") + .setHeader("baz", "quxx") + .build(); + byte[] embedded = converter.embedHeaders(new MessageValues(message), "foo", "baz"); + assertEquals(0xff, embedded[0] & 0xff); + assertEquals("\u0002\u0003foo\u0000\u0000\u0000\u0005\"bar\"\u0003baz\u0000\u0000\u0000\u0006\"quxx\"Hello", + new String(embedded).substring(1)); + + MessageValues extracted = converter.extractHeaders(MessageBuilder.withPayload(embedded).build(), false); + assertEquals("Hello", new String((byte[])extracted.getPayload())); + assertEquals("bar", extracted.get("foo")); + assertEquals("quxx", extracted.get("baz")); + } + + @Test + public void testHeaderEmbeddingMissingHeader() throws Exception { + EmbeddedHeadersMessageConverter converter = new EmbeddedHeadersMessageConverter(); + Message message = MessageBuilder.withPayload("Hello".getBytes()) + .setHeader("foo", "bar") + .build(); + byte[] embedded = converter.embedHeaders(new MessageValues(message), "foo", "baz"); + assertEquals(0xff, embedded[0] & 0xff); + assertEquals("\u0001\u0003foo\u0000\u0000\u0000\u0005\"bar\"Hello", + new String(embedded).substring(1)); + } + + @Test + public void testCanDecodeOldFormat() throws Exception { + EmbeddedHeadersMessageConverter converter = new EmbeddedHeadersMessageConverter(); + byte[] bytes = "\u0002\u0003foo\u0003bar\u0003baz\u0004quxxHello".getBytes("UTF-8"); + Message message = new GenericMessage(bytes); + MessageValues extracted = converter.extractHeaders(message,false); + assertEquals("Hello", new String((byte[])extracted.getPayload())); + assertEquals("bar", extracted.get("foo")); + assertEquals("quxx", extracted.get("baz")); + } + + @Test + public void testBadDecode() throws Exception { + EmbeddedHeadersMessageConverter converter = new EmbeddedHeadersMessageConverter(); + byte[] bytes = "\u0002\u0003foo\u0020bar\u0003baz\u0004quxxHello".getBytes("UTF-8"); + Message message = new GenericMessage(bytes); + try { + converter.extractHeaders(message,false); + Assert.fail("Exception expected"); + } + catch (Exception e) { + String s = EmbeddedHeadersMessageConverter.decodeExceptionMessage(message); + assertThat(e, instanceOf(StringIndexOutOfBoundsException.class)); + assertThat(s, startsWith("Could not convert message: 0203666F6F")); + } + + } + +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-test/README.md b/spring-cloud-streams-bindings/spring-cloud-streams-binding-test/README.md new file mode 100644 index 000000000..b49eeecc2 --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-test/README.md @@ -0,0 +1,2 @@ +Spring XD Test Support +====================== diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-test/pom.xml b/spring-cloud-streams-bindings/spring-cloud-streams-binding-test/pom.xml new file mode 100644 index 000000000..b8888e982 --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-test/pom.xml @@ -0,0 +1,120 @@ + + + 4.0.0 + + org.springframework.cloud + spring-cloud-streams-binding-test + 1.0.0.BUILD-SNAPSHOT + jar + + spring-cloud-streams-binding-test + Test support for binding implementations + + + org.springframework.cloud + spring-cloud-streams-bindings-parent + 1.0.0.BUILD-SNAPSHOT + + + + UTF-8 + 1.4.5.RELEASE + + + + + + + org.springframework.integration + spring-integration-test + ${spring-integration.version} + + + org.apache.avro + avro-compiler + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + org.springframework + spring-web + ${spring-framework.version} + + + + + + + + org.springframework.xd + spring-xd-tuple + ${spring-xd.version} + + + org.springframework.xd + spring-xd-codec + + + test + + + org.springframework.cloud + spring-cloud-streams-binding-spi + + + org.springframework.cloud + spring-cloud-streams-codec + + + diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-test/src/main/java/org/springframework/xd/dirt/integration/bus/AbstractMessageBusTests.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-test/src/main/java/org/springframework/xd/dirt/integration/bus/AbstractMessageBusTests.java new file mode 100644 index 000000000..0eb3ee75f --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-test/src/main/java/org/springframework/xd/dirt/integration/bus/AbstractMessageBusTests.java @@ -0,0 +1,311 @@ +/* + * Copyright 2013-2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.xd.dirt.integration.bus; + +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Properties; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Test; + +import org.springframework.beans.DirectFieldAccessor; +import org.springframework.http.MediaType; +import org.springframework.integration.channel.DirectChannel; +import org.springframework.integration.channel.QueueChannel; +import org.springframework.integration.channel.interceptor.WireTap; +import org.springframework.integration.support.MessageBuilder; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageHeaders; +import org.springframework.xd.dirt.integration.bus.MessageBus.Capability; +import org.springframework.xd.dirt.integration.bus.serializer.MultiTypeCodec; +import org.springframework.xd.dirt.integration.bus.serializer.kryo.PojoCodec; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +/** + * @author Gary Russell + * @author Ilayaperumal Gopinathan + * @author David Turanski + */ +public abstract class AbstractMessageBusTests { + + protected static final Collection ALL = Collections.singletonList(MediaType.ALL); + + protected AbstractTestMessageBus testMessageBus; + + @Test + public void testClean() throws Exception { + MessageBus messageBus = getMessageBus(); + messageBus.bindProducer("foo.0", new DirectChannel(), null); + messageBus.bindConsumer("foo.0", new DirectChannel(), null); + messageBus.bindProducer("foo.1", new DirectChannel(), null); + messageBus.bindConsumer("foo.1", new DirectChannel(), null); + messageBus.bindProducer("foo.2", new DirectChannel(), null); + Collection bindings = getBindings(messageBus); + assertEquals(5, bindings.size()); + messageBus.unbindProducers("foo.0"); + assertEquals(4, bindings.size()); + messageBus.unbindConsumers("foo.0"); + messageBus.unbindProducers("foo.1"); + assertEquals(2, bindings.size()); + messageBus.unbindConsumers("foo.1"); + messageBus.unbindProducers("foo.2"); + assertTrue(bindings.isEmpty()); + } + + @Test + public void testSendAndReceive() throws Exception { + MessageBus messageBus = getMessageBus(); + DirectChannel moduleOutputChannel = new DirectChannel(); + QueueChannel moduleInputChannel = new QueueChannel(); + messageBus.bindProducer("foo.0", moduleOutputChannel, null); + messageBus.bindConsumer("foo.0", moduleInputChannel, null); + Message message = MessageBuilder.withPayload("foo").setHeader(MessageHeaders.CONTENT_TYPE, + "foo/bar").build(); + // Let the consumer actually bind to the producer before sending a msg + busBindUnbindLatency(); + moduleOutputChannel.send(message); + Message inbound = moduleInputChannel.receive(5000); + assertNotNull(inbound); + assertEquals("foo", inbound.getPayload()); + assertNull(inbound.getHeaders().get(XdHeaders.XD_ORIGINAL_CONTENT_TYPE)); + assertEquals("foo/bar", inbound.getHeaders().get(MessageHeaders.CONTENT_TYPE)); + messageBus.unbindProducers("foo.0"); + messageBus.unbindConsumers("foo.0"); + } + + @Test + public void testSendAndReceiveNoOriginalContentType() throws Exception { + MessageBus messageBus = getMessageBus(); + DirectChannel moduleOutputChannel = new DirectChannel(); + QueueChannel moduleInputChannel = new QueueChannel(); + messageBus.bindProducer("bar.0", moduleOutputChannel, null); + messageBus.bindConsumer("bar.0", moduleInputChannel, null); + busBindUnbindLatency(); + + Message message = MessageBuilder.withPayload("foo").build(); + moduleOutputChannel.send(message); + Message inbound = moduleInputChannel.receive(5000); + assertNotNull(inbound); + assertEquals("foo", inbound.getPayload()); + assertNull(inbound.getHeaders().get(XdHeaders.XD_ORIGINAL_CONTENT_TYPE)); + assertNull(inbound.getHeaders().get(MessageHeaders.CONTENT_TYPE)); + messageBus.unbindProducers("bar.0"); + messageBus.unbindConsumers("bar.0"); + } + + @Test + public void testSendAndReceivePubSub() throws Exception { + MessageBus messageBus = getMessageBus(); + DirectChannel moduleOutputChannel = new DirectChannel(); + // Test pub/sub by emulating how StreamPlugin handles taps + DirectChannel tapChannel = new DirectChannel(); + QueueChannel moduleInputChannel = new QueueChannel(); + QueueChannel module2InputChannel = new QueueChannel(); + QueueChannel module3InputChannel = new QueueChannel(); + messageBus.bindProducer("baz.0", moduleOutputChannel, null); + messageBus.bindConsumer("baz.0", moduleInputChannel, null); + moduleOutputChannel.addInterceptor(new WireTap(tapChannel)); + messageBus.bindPubSubProducer("tap:baz.http", tapChannel, null); + // A new module is using the tap as an input channel + String fooTapName = messageBus.isCapable(Capability.DURABLE_PUBSUB) ? "foo.tap:baz.http" : "tap:baz.http"; + messageBus.bindPubSubConsumer(fooTapName, module2InputChannel, null); + // Another new module is using tap as an input channel + String barTapName = messageBus.isCapable(Capability.DURABLE_PUBSUB) ? "bar.tap:baz.http" : "tap:baz.http"; + messageBus.bindPubSubConsumer(barTapName, module3InputChannel, null); + Message message = MessageBuilder.withPayload("foo").setHeader(MessageHeaders.CONTENT_TYPE, + "foo/bar").build(); + boolean success = false; + boolean retried = false; + while (!success) { + moduleOutputChannel.send(message); + Message inbound = moduleInputChannel.receive(5000); + assertNotNull(inbound); + assertEquals("foo", inbound.getPayload()); + assertNull(inbound.getHeaders().get(XdHeaders.XD_ORIGINAL_CONTENT_TYPE)); + assertEquals("foo/bar", inbound.getHeaders().get(MessageHeaders.CONTENT_TYPE)); + Message tapped1 = module2InputChannel.receive(5000); + Message tapped2 = module3InputChannel.receive(5000); + if (tapped1 == null || tapped2 == null) { + // listener may not have started + assertFalse("Failed to receive tap after retry", retried); + retried = true; + continue; + } + success = true; + assertEquals("foo", tapped1.getPayload()); + assertNull(tapped1.getHeaders().get(XdHeaders.XD_ORIGINAL_CONTENT_TYPE)); + assertEquals("foo/bar", tapped1.getHeaders().get(MessageHeaders.CONTENT_TYPE)); + assertEquals("foo", tapped2.getPayload()); + assertNull(tapped2.getHeaders().get(XdHeaders.XD_ORIGINAL_CONTENT_TYPE)); + assertEquals("foo/bar", tapped2.getHeaders().get(MessageHeaders.CONTENT_TYPE)); + } + // delete one tap stream is deleted + messageBus.unbindConsumer(barTapName, module3InputChannel); + Message message2 = MessageBuilder.withPayload("bar").setHeader(MessageHeaders.CONTENT_TYPE, + "foo/bar").build(); + moduleOutputChannel.send(message2); + + // other tap still receives messages + Message tapped = module2InputChannel.receive(5000); + assertNotNull(tapped); + + // Removed tap does not + assertNull(module3InputChannel.receive(1000)); + + // when other tap stream is deleted + messageBus.unbindConsumer(fooTapName, module2InputChannel); + // Clean up as StreamPlugin would + messageBus.unbindConsumer("baz.0", moduleInputChannel); + messageBus.unbindProducer("baz.0", moduleOutputChannel); + messageBus.unbindProducers("tap:baz.http"); + assertTrue(getBindings(messageBus).isEmpty()); + } + + @Test + public void createInboundPubSubBeforeOutboundPubSub() throws Exception { + MessageBus messageBus = getMessageBus(); + DirectChannel moduleOutputChannel = new DirectChannel(); + // Test pub/sub by emulating how StreamPlugin handles taps + DirectChannel tapChannel = new DirectChannel(); + QueueChannel moduleInputChannel = new QueueChannel(); + QueueChannel module2InputChannel = new QueueChannel(); + QueueChannel module3InputChannel = new QueueChannel(); + // Create the tap first + String fooTapName = messageBus.isCapable(Capability.DURABLE_PUBSUB) ? "foo.tap:baz.http" : "tap:baz.http"; + messageBus.bindPubSubConsumer(fooTapName, module2InputChannel, null); + + // Then create the stream + messageBus.bindProducer("baz.0", moduleOutputChannel, null); + messageBus.bindConsumer("baz.0", moduleInputChannel, null); + moduleOutputChannel.addInterceptor(new WireTap(tapChannel)); + messageBus.bindPubSubProducer("tap:baz.http", tapChannel, null); + + // Another new module is using tap as an input channel + String barTapName = messageBus.isCapable(Capability.DURABLE_PUBSUB) ? "bar.tap:baz.http" : "tap:baz.http"; + messageBus.bindPubSubConsumer(barTapName, module3InputChannel, null); + Message message = MessageBuilder.withPayload("foo").setHeader(MessageHeaders.CONTENT_TYPE, + "foo/bar").build(); + boolean success = false; + boolean retried = false; + while (!success) { + moduleOutputChannel.send(message); + Message inbound = moduleInputChannel.receive(5000); + assertNotNull(inbound); + assertEquals("foo", inbound.getPayload()); + assertNull(inbound.getHeaders().get(XdHeaders.XD_ORIGINAL_CONTENT_TYPE)); + assertEquals("foo/bar", inbound.getHeaders().get(MessageHeaders.CONTENT_TYPE)); + Message tapped1 = module2InputChannel.receive(5000); + Message tapped2 = module3InputChannel.receive(5000); + if (tapped1 == null || tapped2 == null) { + // listener may not have started + assertFalse("Failed to receive tap after retry", retried); + retried = true; + continue; + } + success = true; + assertEquals("foo", tapped1.getPayload()); + assertNull(tapped1.getHeaders().get(XdHeaders.XD_ORIGINAL_CONTENT_TYPE)); + assertEquals("foo/bar", tapped1.getHeaders().get(MessageHeaders.CONTENT_TYPE)); + assertEquals("foo", tapped2.getPayload()); + assertNull(tapped2.getHeaders().get(XdHeaders.XD_ORIGINAL_CONTENT_TYPE)); + assertEquals("foo/bar", tapped2.getHeaders().get(MessageHeaders.CONTENT_TYPE)); + } + // delete one tap stream is deleted + messageBus.unbindConsumer(barTapName, module3InputChannel); + Message message2 = MessageBuilder.withPayload("bar").setHeader(MessageHeaders.CONTENT_TYPE, + "foo/bar").build(); + moduleOutputChannel.send(message2); + + // other tap still receives messages + Message tapped = module2InputChannel.receive(5000); + assertNotNull(tapped); + + // Removed tap does not + assertNull(module3InputChannel.receive(1000)); + + // when other tap stream is deleted + messageBus.unbindConsumer(fooTapName, module2InputChannel); + // Clean up as StreamPlugin would + messageBus.unbindConsumer("baz.0", moduleInputChannel); + messageBus.unbindProducer("baz.0", moduleOutputChannel); + messageBus.unbindProducers("tap:baz.http"); + assertTrue(getBindings(messageBus).isEmpty()); + } + + @Test + public void testBadDynamic() throws Exception { + Properties properties = new Properties(); + properties.setProperty(BusProperties.PARTITION_KEY_EXPRESSION, "'foo'"); + MessageBus messageBus = getMessageBus(); + try { + messageBus.bindDynamicProducer("queue:foo", properties); + fail("Exception expected"); + } + catch (MessageBusException mbe) { + Assert.assertEquals("Failed to bind dynamic channel 'queue:foo' with properties " + + "{partitionKeyExpression='foo'}", + mbe.getMessage()); + if (messageBus instanceof AbstractTestMessageBus) { + messageBus = ((AbstractTestMessageBus) messageBus).getCoreMessageBus(); + } + assertFalse(((MessageBusSupport) messageBus).getApplicationContext().containsBean("queue:foo")); + } + } + + protected Collection getBindings(MessageBus testMessageBus) { + if (testMessageBus instanceof AbstractTestMessageBus) { + return getBindingsFromMsgBus(((AbstractTestMessageBus) testMessageBus).getCoreMessageBus()); + } + return Collections.EMPTY_LIST; + } + + protected Collection getBindingsFromMsgBus(MessageBus messageBus) { + DirectFieldAccessor accessor = new DirectFieldAccessor(messageBus); + return (List) accessor.getPropertyValue("bindings"); + } + + @SuppressWarnings({"unchecked", "rawtypes"}) + protected MultiTypeCodec getCodec() { + return new PojoCodec(); + } + + protected abstract MessageBus getMessageBus() throws Exception; + + @After + public void cleanup() { + if (testMessageBus != null) { + testMessageBus.cleanup(); + } + } + + /** + * If appropriate, let the bus middleware settle down a bit while binding/unbinding actually happens. + */ + protected void busBindUnbindLatency() throws InterruptedException { + // default none + } + +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-test/src/main/java/org/springframework/xd/dirt/integration/bus/AbstractTestMessageBus.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-test/src/main/java/org/springframework/xd/dirt/integration/bus/AbstractTestMessageBus.java new file mode 100644 index 000000000..3db9fdb9a --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-test/src/main/java/org/springframework/xd/dirt/integration/bus/AbstractTestMessageBus.java @@ -0,0 +1,141 @@ +/* + * Copyright 2014-2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.xd.dirt.integration.bus; + +import java.util.HashSet; +import java.util.Properties; +import java.util.Set; + +import org.springframework.expression.spel.support.StandardEvaluationContext; +import org.springframework.messaging.MessageChannel; + + +/** + * Abstract class that adds test support for {@link MessageBus}. + * + * @author Ilayaperumal Gopinathan + * @author Gary Russell + */ +public abstract class AbstractTestMessageBus implements MessageBus { + + protected Set queues = new HashSet(); + + protected Set topics = new HashSet(); + + private C messageBus; + + public void setMessageBus(C messageBus) { + messageBus.setIntegrationEvaluationContext(new StandardEvaluationContext()); + try { + messageBus.afterPropertiesSet(); + } + catch (Exception e) { + throw new RuntimeException("Failed to initialize message bus", e); + } + this.messageBus = messageBus; + } + + @Override + public void bindConsumer(String name, MessageChannel moduleInputChannel, Properties properties) { + messageBus.bindConsumer(name, moduleInputChannel, properties); + queues.add(name); + } + + @Override + public void bindPubSubConsumer(String name, MessageChannel inputChannel, Properties properties) { + messageBus.bindPubSubConsumer(name, inputChannel, properties); + addTopic(name); + } + + @Override + public void bindProducer(String name, MessageChannel moduleOutputChannel, Properties properties) { + messageBus.bindProducer(name, moduleOutputChannel, properties); + queues.add(name); + } + + @Override + public void bindPubSubProducer(String name, MessageChannel outputChannel, Properties properties) { + messageBus.bindPubSubProducer(name, outputChannel, properties); + addTopic(name); + } + + @Override + public void bindRequestor(String name, MessageChannel requests, MessageChannel replies, + Properties properties) { + messageBus.bindRequestor(name, requests, replies, properties); + queues.add(name + ".requests"); + } + + @Override + public void bindReplier(String name, MessageChannel requests, MessageChannel replies, + Properties properties) { + messageBus.bindReplier(name, requests, replies, properties); + queues.add(name + ".requests"); + } + + private void addTopic(String topicName) { + topics.add("topic." + topicName); + } + + public C getCoreMessageBus() { + return messageBus; + } + + public abstract void cleanup(); + + @Override + public void unbindConsumers(String name) { + messageBus.unbindConsumers(name); + } + + @Override + public void unbindProducers(String name) { + messageBus.unbindProducers(name); + } + + @Override + public void unbindConsumer(String name, MessageChannel channel) { + messageBus.unbindConsumer(name, channel); + } + + @Override + public void unbindProducer(String name, MessageChannel channel) { + messageBus.unbindProducer(name, channel); + } + + @Override + public MessageChannel bindDynamicProducer(String name, Properties properties) { + this.queues.add(name); + return this.messageBus.bindDynamicProducer(name, properties); + } + + @Override + public MessageChannel bindDynamicPubSubProducer(String name, Properties properties) { + this.topics.add(name); + return this.messageBus.bindDynamicPubSubProducer(name, properties); + } + + @Override + public boolean isCapable(Capability capability) { + return this.messageBus.isCapable(capability); + } + + public MessageBus getMessageBus() { + return this.messageBus; + } + +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-test/src/main/java/org/springframework/xd/dirt/integration/bus/BrokerBusTests.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-test/src/main/java/org/springframework/xd/dirt/integration/bus/BrokerBusTests.java new file mode 100644 index 000000000..428eb8010 --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-test/src/main/java/org/springframework/xd/dirt/integration/bus/BrokerBusTests.java @@ -0,0 +1,114 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.xd.dirt.integration.bus; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; + +import java.util.Properties; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import org.junit.Test; + +import org.springframework.integration.channel.DirectChannel; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageHandler; +import org.springframework.messaging.MessagingException; +import org.springframework.messaging.support.GenericMessage; + + +/** + * Tests for buses that use an external broker. + * + * @author Gary Russell + */ +public abstract class BrokerBusTests extends +AbstractMessageBusTests { + + @Test + public void testDirectBinding() throws Exception { + MessageBus bus = getMessageBus(); + Properties properties = new Properties(); + properties.setProperty(BusProperties.DIRECT_BINDING_ALLOWED, "true"); + + DirectChannel moduleInputChannel = new DirectChannel(); + moduleInputChannel.setBeanName("direct.input"); + DirectChannel moduleOutputChannel = new DirectChannel(); + moduleOutputChannel.setBeanName("direct.output"); + bus.bindConsumer("direct.0", moduleInputChannel, null); + bus.bindProducer("direct.0", moduleOutputChannel, properties); + + final AtomicReference caller = new AtomicReference(); + final AtomicInteger count = new AtomicInteger(); + moduleInputChannel.subscribe(new MessageHandler() { + + @Override + public void handleMessage(Message message) throws MessagingException { + caller.set(Thread.currentThread()); + count.incrementAndGet(); + } + }); + + moduleOutputChannel.send(new GenericMessage("foo")); + moduleOutputChannel.send(new GenericMessage("foo")); + + assertNotNull(caller.get()); + assertSame(Thread.currentThread(), caller.get()); + assertEquals(2, count.get()); + assertNull(spyOn("direct.0").receive(true)); + + // Remove direct binding and bind producer to the bus + bus.unbindConsumers("direct.0"); + busBindUnbindLatency(); + + Spy spy = spyOn("direct.0"); + count.set(0); + moduleOutputChannel.send(new GenericMessage("bar")); + moduleOutputChannel.send(new GenericMessage("baz")); + Object bar = spy.receive(false); + assertEquals("bar", bar); + Object baz = spy.receive(false); + assertEquals("baz", baz); + assertEquals(0, count.get()); + + // Unbind producer from bus and bind directly again + caller.set(null); + bus.bindConsumer("direct.0", moduleInputChannel, null); + moduleOutputChannel.send(new GenericMessage("foo")); + moduleOutputChannel.send(new GenericMessage("foo")); + assertNotNull(caller.get()); + assertSame(Thread.currentThread(), caller.get()); + assertEquals(2, count.get()); + assertNull(spy.receive(true)); + + bus.unbindProducers("direct.0"); + bus.unbindConsumers("direct.0"); + } + + /** + * Create a new spy on the given 'queue'. This allows de-correlating the creation of + * the 'connection' from its actual usage, which may be needed by some implementations to + * see messages sent after connection creation. + */ + public abstract Spy spyOn(final String name); + + + +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-test/src/main/java/org/springframework/xd/dirt/integration/bus/BusTestUtils.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-test/src/main/java/org/springframework/xd/dirt/integration/bus/BusTestUtils.java new file mode 100644 index 000000000..4da314c40 --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-test/src/main/java/org/springframework/xd/dirt/integration/bus/BusTestUtils.java @@ -0,0 +1,49 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.xd.dirt.integration.bus; + +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; + +import org.springframework.beans.factory.config.ConfigurableListableBeanFactory; +import org.springframework.context.support.AbstractApplicationContext; +import org.springframework.integration.support.DefaultMessageBuilderFactory; +import org.springframework.integration.support.MessageBuilderFactory; +import org.springframework.integration.support.utils.IntegrationUtils; + + +/** + * + * @author Gary Russell + */ +public class BusTestUtils { + + private static final MessageBuilderFactory mbf = new DefaultMessageBuilderFactory(); + + public static final AbstractApplicationContext MOCK_AC = mock(AbstractApplicationContext.class); + + public static final ConfigurableListableBeanFactory MOCK_BF = mock(ConfigurableListableBeanFactory.class); + + static { + when(MOCK_BF.getBean(IntegrationUtils.INTEGRATION_MESSAGE_BUILDER_FACTORY_BEAN_NAME, + MessageBuilderFactory.class)).thenReturn(mbf); + when(MOCK_AC.getBean(IntegrationUtils.INTEGRATION_MESSAGE_BUILDER_FACTORY_BEAN_NAME, + MessageBuilderFactory.class)).thenReturn(mbf); + when(MOCK_AC.getBeanFactory()).thenReturn(MOCK_BF); + } + +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-test/src/main/java/org/springframework/xd/dirt/integration/bus/PartitionCapableBusTests.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-test/src/main/java/org/springframework/xd/dirt/integration/bus/PartitionCapableBusTests.java new file mode 100644 index 000000000..29777cce2 --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-test/src/main/java/org/springframework/xd/dirt/integration/bus/PartitionCapableBusTests.java @@ -0,0 +1,271 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.xd.dirt.integration.bus; + +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.containsInAnyOrder; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasProperty; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertThat; + +import java.util.Arrays; +import java.util.List; +import java.util.Properties; + +import org.hamcrest.CustomMatcher; +import org.hamcrest.Matcher; +import org.hamcrest.Matchers; +import org.junit.Test; + +import org.springframework.integration.IntegrationMessageHeaderAccessor; +import org.springframework.integration.channel.DirectChannel; +import org.springframework.integration.channel.QueueChannel; +import org.springframework.integration.endpoint.AbstractEndpoint; +import org.springframework.integration.support.MessageBuilder; +import org.springframework.messaging.Message; +import org.springframework.messaging.support.GenericMessage; +import org.springframework.xd.test.TestUtils; + + +/** + * Tests for buses that support partitioning. + * + * @author Gary Russell + */ +abstract public class PartitionCapableBusTests extends BrokerBusTests { + + @Test + public void testBadProperties() throws Exception { + MessageBus bus = getMessageBus(); + Properties properties = new Properties(); + properties.put("foo", "bar"); + properties.put("baz", "qux"); + + DirectChannel output = new DirectChannel(); + try { + bus.bindProducer("badprops.0", output, properties); + } + catch (IllegalArgumentException e) { + assertThat(e.getMessage(), allOf(Matchers.containsString(bus.getClass().getSimpleName().replace("Test", "") + + " does not support producer "), + containsString("foo"), + containsString("baz"), + containsString(" for badprops.0."))); + } + + properties.remove("baz"); + try { + bus.bindConsumer("badprops.0", output, properties); + } + catch (IllegalArgumentException e) { + assertThat(e.getMessage(), equalTo(bus.getClass().getSimpleName().replace("Test", "") + + " does not support consumer property: foo for badprops.0.")); + } + } + + @Test + public void testPartitionedModuleSpEL() throws Exception { + MessageBus bus = getMessageBus(); + Properties properties = new Properties(); + properties.put("partitionKeyExpression", "payload"); + properties.put("partitionSelectorExpression", "hashCode()"); + properties.put(BusProperties.NEXT_MODULE_COUNT, "3"); + properties.put(BusProperties.NEXT_MODULE_CONCURRENCY, "2"); + + DirectChannel output = new DirectChannel(); + output.setBeanName("test.output"); + bus.bindProducer("part.0", output, properties); + @SuppressWarnings("unchecked") + List bindings = TestUtils.getPropertyValue(bus, "messageBus.bindings", List.class); + assertEquals(1, bindings.size()); + try { + AbstractEndpoint endpoint = bindings.get(0).getEndpoint(); + assertThat(getEndpointRouting(endpoint), containsString("part.0-' + headers['partition']")); + } + catch (UnsupportedOperationException ignored) { + + } + + properties.clear(); + properties.put("concurrency", "2"); + properties.put("partitionIndex", "0"); + properties.put("count","3"); + QueueChannel input0 = new QueueChannel(); + input0.setBeanName("test.input0S"); + bus.bindConsumer("part.0", input0, properties); + properties.put("partitionIndex", "1"); + QueueChannel input1 = new QueueChannel(); + input1.setBeanName("test.input1S"); + bus.bindConsumer("part.0", input1, properties); + properties.put("partitionIndex", "2"); + QueueChannel input2 = new QueueChannel(); + input2.setBeanName("test.input2S"); + bus.bindConsumer("part.0", input2, properties); + + Message message2 = MessageBuilder.withPayload(2) + .setHeader(IntegrationMessageHeaderAccessor.CORRELATION_ID, "foo") + .setHeader(IntegrationMessageHeaderAccessor.SEQUENCE_NUMBER, 42) + .setHeader(IntegrationMessageHeaderAccessor.SEQUENCE_SIZE, 43) + .setHeader("xdReplyChannel", "bar") + .build(); + output.send(message2); + output.send(new GenericMessage(1)); + output.send(new GenericMessage(0)); + + Message receive0 = input0.receive(1000); + assertNotNull(receive0); + Message receive1 = input1.receive(1000); + assertNotNull(receive1); + Message receive2 = input2.receive(1000); + assertNotNull(receive2); + + Matcher> fooMatcher = new CustomMatcher>("the message with 'foo' as its correlationId") { + + @Override + public boolean matches(Object item) { + IntegrationMessageHeaderAccessor accessor = new IntegrationMessageHeaderAccessor((Message) item); + boolean result = "foo".equals(accessor.getCorrelationId()) && + 42 == accessor.getSequenceNumber() && + 43 == accessor.getSequenceSize() && + "bar".equals(accessor.getHeader("xdReplyChannel")); + return result; + } + }; + + if (usesExplicitRouting()) { + assertEquals(0, receive0.getPayload()); + assertEquals(1, receive1.getPayload()); + assertEquals(2, receive2.getPayload()); + + assertThat(receive2, fooMatcher); + + } + else { + + assertThat(Arrays.asList( + (Integer) receive0.getPayload(), + (Integer) receive1.getPayload(), + (Integer) receive2.getPayload()), + containsInAnyOrder(0, 1, 2)); + + @SuppressWarnings("unchecked") + Matcher>> containsOur3Messages = containsInAnyOrder( + fooMatcher, + hasProperty("payload", equalTo(0)), + hasProperty("payload", equalTo(1)) + ); + assertThat( + Arrays.asList(receive0, receive1, receive2), + containsOur3Messages); + + } + + bus.unbindConsumers("part.0"); + bus.unbindProducers("part.0"); + } + + @Test + public void testPartitionedModuleJava() throws Exception { + MessageBus bus = getMessageBus(); + Properties properties = new Properties(); + properties.put("partitionKeyExtractorClass", "org.springframework.xd.dirt.integration.bus.PartitionTestSupport"); + properties.put("partitionSelectorClass", "org.springframework.xd.dirt.integration.bus.PartitionTestSupport"); + properties.put(BusProperties.NEXT_MODULE_COUNT, "3"); + properties.put(BusProperties.NEXT_MODULE_CONCURRENCY, "2"); + + DirectChannel output = new DirectChannel(); + output.setBeanName("test.output"); + bus.bindProducer("partJ.0", output, properties); + @SuppressWarnings("unchecked") + List bindings = TestUtils.getPropertyValue(bus, "messageBus.bindings", List.class); + assertEquals(1, bindings.size()); + if (usesExplicitRouting()) { + AbstractEndpoint endpoint = bindings.get(0).getEndpoint(); + assertThat(getEndpointRouting(endpoint), containsString("partJ.0-' + headers['partition']")); + } + + properties.clear(); + properties.put("concurrency", "2"); + properties.put("count","3"); + properties.put("partitionIndex", "0"); + QueueChannel input0 = new QueueChannel(); + input0.setBeanName("test.input0J"); + bus.bindConsumer("partJ.0", input0, properties); + properties.put("partitionIndex", "1"); + QueueChannel input1 = new QueueChannel(); + input1.setBeanName("test.input1J"); + bus.bindConsumer("partJ.0", input1, properties); + properties.put("partitionIndex", "2"); + QueueChannel input2 = new QueueChannel(); + input2.setBeanName("test.input2J"); + bus.bindConsumer("partJ.0", input2, properties); + + output.send(new GenericMessage(2)); + output.send(new GenericMessage(1)); + output.send(new GenericMessage(0)); + + Message receive0 = input0.receive(1000); + assertNotNull(receive0); + Message receive1 = input1.receive(1000); + assertNotNull(receive1); + Message receive2 = input2.receive(1000); + assertNotNull(receive2); + + if (usesExplicitRouting()) { + assertEquals(0, receive0.getPayload()); + assertEquals(1, receive1.getPayload()); + assertEquals(2, receive2.getPayload()); + } + else { + + assertThat(Arrays.asList( + (Integer) receive0.getPayload(), + (Integer) receive1.getPayload(), + (Integer) receive2.getPayload()), + containsInAnyOrder(0, 1, 2)); + } + + bus.unbindConsumers("partJ.0"); + bus.unbindProducers("partJ.0"); + } + + /** + * Implementations should return whether the bus under test uses "explicit" routing (e.g. Rabbit) + * whereby XD is responsible for assigning a partition and knows which exact consumer will receive the + * message (i.e. honor "partitionIndex") or "implicit" routing (e.g. Kafka) whereby the only guarantee + * is that messages will be spread, but we don't control exactly which consumer gets which message. + */ + protected abstract boolean usesExplicitRouting(); + + /** + * For implementations that rely on explicit routing, return the routing expression. + */ + protected String getEndpointRouting(AbstractEndpoint endpoint) { + throw new UnsupportedOperationException(); + } + + /** + * For implementations that rely on explicit routing, return the routing expression. + */ + protected String getPubSubEndpointRouting(AbstractEndpoint endpoint) { + throw new UnsupportedOperationException(); + } + +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-test/src/main/java/org/springframework/xd/dirt/integration/bus/PartitionTestSupport.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-test/src/main/java/org/springframework/xd/dirt/integration/bus/PartitionTestSupport.java new file mode 100644 index 000000000..fbd982a01 --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-test/src/main/java/org/springframework/xd/dirt/integration/bus/PartitionTestSupport.java @@ -0,0 +1,38 @@ +/* + * Copyright 2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.xd.dirt.integration.bus; + +import org.springframework.messaging.Message; + + +/** + * + * @author Gary Russell + */ +public class PartitionTestSupport implements PartitionKeyExtractorStrategy, PartitionSelectorStrategy { + + @Override + public int selectPartition(Object key, int divisor) { + return key.hashCode() % divisor; + } + + @Override + public Object extractKey(Message message) { + return message.getPayload(); + } + +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-test/src/main/java/org/springframework/xd/dirt/integration/bus/Spy.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-test/src/main/java/org/springframework/xd/dirt/integration/bus/Spy.java new file mode 100644 index 000000000..d866a1c0c --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-test/src/main/java/org/springframework/xd/dirt/integration/bus/Spy.java @@ -0,0 +1,13 @@ +package org.springframework.xd.dirt.integration.bus; + +/** + * Represents an out-of-band connection to the underlying middleware, + * so that tests can check that some messages actually do (or do not) + * transit through it. + * + * @author Eric Bottard + */ +public interface Spy { + + public Object receive(boolean expectNull) throws Exception; +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-test/src/main/java/org/springframework/xd/test/AbstractExternalResourceTestSupport.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-test/src/main/java/org/springframework/xd/test/AbstractExternalResourceTestSupport.java new file mode 100644 index 000000000..668911c9f --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-test/src/main/java/org/springframework/xd/test/AbstractExternalResourceTestSupport.java @@ -0,0 +1,134 @@ +/* + * Copyright 2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.xd.test; + +import static org.junit.Assert.fail; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.junit.Assume; +import org.junit.Rule; +import org.junit.rules.TestRule; +import org.junit.runner.Description; +import org.junit.runners.model.Statement; + +import org.springframework.util.Assert; + + +/** + * Abstract base class for JUnit {@link Rule}s that detect the presence of some external resource. If the resource is + * indeed present, it will be available during the test lifecycle through {@link #getResource()}. If it is not, tests + * will either fail or be skipped, depending on the value of system property {@value #XD_EXTERNAL_SERVERS_REQUIRED}. + * + * @author Eric Bottard + * @author Gary Russell + */ +public abstract class AbstractExternalResourceTestSupport implements TestRule { + + public static final String XD_EXTERNAL_SERVERS_REQUIRED = "XD_EXTERNAL_SERVERS_REQUIRED"; + + protected R resource; + + private String resourceDescription; + + protected final Logger logger = LoggerFactory.getLogger(this.getClass()); + + protected AbstractExternalResourceTestSupport(String resourceDescription) { + Assert.hasText(resourceDescription, "resourceDescription is required"); + this.resourceDescription = resourceDescription; + } + + @Override + public Statement apply(final Statement base, Description description) { + try { + obtainResource(); + } + catch (Exception e) { + maybeCleanup(); + + return failOrSkip(e); + } + + return new Statement() { + + @Override + public void evaluate() throws Throwable { + try { + base.evaluate(); + } + finally { + try { + cleanupResource(); + } + catch (Exception ignored) { + logger.warn("Exception while trying to cleanup proper resource", ignored); + } + } + } + + }; + } + + private Statement failOrSkip(final Exception e) { + String serversRequired = System.getenv(XD_EXTERNAL_SERVERS_REQUIRED); + if ("true".equalsIgnoreCase(serversRequired)) { + logger.error(resourceDescription + " IS REQUIRED BUT NOT AVAILABLE", e); + fail(resourceDescription + " IS NOT AVAILABLE"); + // Never reached, here to satisfy method signature + return null; + } + else { + logger.error(resourceDescription + " IS NOT AVAILABLE, SKIPPING TESTS", e); + return new Statement() { + + @Override + public void evaluate() throws Throwable { + Assume.assumeTrue("Skipping test due to " + resourceDescription + " not being available " + e, false); + } + }; + } + } + + private void maybeCleanup() { + if (resource != null) { + try { + cleanupResource(); + } + catch (Exception ignored) { + logger.warn("Exception while trying to cleanup failed resource", ignored); + } + } + } + + public R getResource() { + return resource; + } + + /** + * Perform cleanup of the {@link #resource} field, which is guaranteed to be non null. + * + * @throws Exception any exception thrown by this method will be logged and swallowed + */ + protected abstract void cleanupResource() throws Exception; + + /** + * Try to obtain and validate a resource. Implementors should either set the {@link #resource} field with a valid + * resource and return normally, or throw an exception. + */ + protected abstract void obtainResource() throws Exception; + +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-test/src/main/java/org/springframework/xd/test/TestUtils.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-test/src/main/java/org/springframework/xd/test/TestUtils.java new file mode 100644 index 000000000..71c807999 --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-test/src/main/java/org/springframework/xd/test/TestUtils.java @@ -0,0 +1,60 @@ +/* + * Copyright 2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.xd.test; + +import org.springframework.beans.DirectFieldAccessor; +import org.springframework.util.Assert; + +/** + * Copy of class in org.springframework.amqp.utils.test to avoid dependency on spring-amqp + */ +public class TestUtils { + + /** + * Uses nested {@link DirectFieldAccessor}s to obtain a property using dotted notation to traverse fields; e.g. + * "foo.bar.baz" will obtain a reference to the baz field of the bar field of foo. Adopted from Spring Integration. + * @param root The object. + * @param propertyPath The path. + * @return The field. + */ + public static Object getPropertyValue(Object root, String propertyPath) { + Object value = null; + DirectFieldAccessor accessor = new DirectFieldAccessor(root); + String[] tokens = propertyPath.split("\\."); + for (int i = 0; i < tokens.length; i++) { + value = accessor.getPropertyValue(tokens[i]); + if (value != null) { + accessor = new DirectFieldAccessor(value); + } + else if (i == tokens.length - 1) { + return null; + } + else { + throw new IllegalArgumentException("intermediate property '" + tokens[i] + "' is null"); + } + } + return value; + } + + @SuppressWarnings("unchecked") + public static T getPropertyValue(Object root, String propertyPath, Class type) { + Object value = getPropertyValue(root, propertyPath); + if (value != null) { + Assert.isAssignable(type, value.getClass()); + } + return (T) value; + } +} diff --git a/spring-cloud-streams-bindings/spring-cloud-streams-binding-test/src/test/java/org/springframework/xd/dirt/integration/bus/MessageBusSupportTests.java b/spring-cloud-streams-bindings/spring-cloud-streams-binding-test/src/test/java/org/springframework/xd/dirt/integration/bus/MessageBusSupportTests.java new file mode 100644 index 000000000..9495a8b0b --- /dev/null +++ b/spring-cloud-streams-bindings/spring-cloud-streams-binding-test/src/test/java/org/springframework/xd/dirt/integration/bus/MessageBusSupportTests.java @@ -0,0 +1,302 @@ +/* + * Copyright 2013 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.xd.dirt.integration.bus; + +import java.io.IOException; +import java.util.Collections; +import java.util.Properties; + +import org.junit.Before; +import org.junit.Test; + +import org.springframework.integration.support.MessageBuilder; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageChannel; +import org.springframework.messaging.MessageHeaders; +import org.springframework.messaging.converter.ContentTypeResolver; +import org.springframework.messaging.support.GenericMessage; +import org.springframework.util.MimeType; +import org.springframework.util.MimeTypeUtils; +import org.springframework.xd.dirt.integration.bus.MessageBusSupport.JavaClassMimeTypeConversion; +import org.springframework.xd.dirt.integration.bus.serializer.kryo.PojoCodec; +import org.springframework.xd.tuple.DefaultTuple; +import org.springframework.xd.tuple.Tuple; +import org.springframework.xd.tuple.TupleBuilder; +import org.springframework.xd.tuple.serializer.kryo.TupleKryoRegistrar; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertSame; + +/** + * @author Gary Russell + * @author David Turanski + */ +public class MessageBusSupportTests { + + private ContentTypeResolver contentTypeResolver = new StringConvertingContentTypeResolver(); + + private final TestMessageBus messageBus = new TestMessageBus(); + + @SuppressWarnings({"unchecked", "rawtypes"}) + @Before + public void setUp() { + messageBus.setCodec(new PojoCodec(new TupleKryoRegistrar())); + } + + @Test + public void testBytesPassThru() { + byte[] payload = "foo".getBytes(); + Message message = MessageBuilder.withPayload(payload).build(); + MessageValues converted = messageBus.serializePayloadIfNecessary(message + ); + assertSame(payload, converted.getPayload()); + Message convertedMessage = converted.toMessage(); + assertSame(payload, convertedMessage.getPayload()); + assertEquals(MimeTypeUtils.APPLICATION_OCTET_STREAM, + contentTypeResolver.resolve(convertedMessage.getHeaders())); + MessageValues reconstructed = messageBus.deserializePayloadIfNecessary(convertedMessage); + payload = (byte[]) reconstructed.getPayload(); + assertSame(converted.getPayload(), payload); + assertNull(reconstructed.get(XdHeaders.XD_ORIGINAL_CONTENT_TYPE)); + } + + @Test + public void testBytesPassThruContentType() { + byte[] payload = "foo".getBytes(); + Message message = MessageBuilder.withPayload(payload) + .setHeader(MessageHeaders.CONTENT_TYPE, MimeTypeUtils.APPLICATION_OCTET_STREAM_VALUE) + .build(); + MessageValues messageValues = messageBus.serializePayloadIfNecessary(message + ); + Message converted = messageValues.toMessage(); + assertSame(payload, converted.getPayload()); + assertEquals(MimeTypeUtils.APPLICATION_OCTET_STREAM, + contentTypeResolver.resolve(converted.getHeaders())); + MessageValues reconstructed = messageBus.deserializePayloadIfNecessary(converted); + payload = (byte[]) reconstructed.getPayload(); + assertSame(converted.getPayload(), payload); + assertEquals(MimeTypeUtils.APPLICATION_OCTET_STREAM_VALUE, + reconstructed.get(MessageHeaders.CONTENT_TYPE)); + assertNull(reconstructed.get(XdHeaders.XD_ORIGINAL_CONTENT_TYPE)); + } + + @Test + public void testString() throws IOException { + MessageValues convertedValues = messageBus.serializePayloadIfNecessary( + new GenericMessage("foo")); + + Message converted = convertedValues.toMessage(); + assertEquals(MimeTypeUtils.TEXT_PLAIN, + contentTypeResolver.resolve(converted.getHeaders())); + MessageValues reconstructed = messageBus.deserializePayloadIfNecessary(converted); + assertEquals("foo", reconstructed.getPayload()); + assertNull(reconstructed.get(MessageHeaders.CONTENT_TYPE)); + } + + @Test + public void testContentTypePreserved() throws IOException { + Message inbound = MessageBuilder.withPayload("{\"foo\":\"foo\"}") + .copyHeaders(Collections.singletonMap(MessageHeaders.CONTENT_TYPE, MimeTypeUtils.APPLICATION_JSON)) + .build(); + MessageValues convertedValues = messageBus.serializePayloadIfNecessary( + inbound); + + Message converted = convertedValues.toMessage(); + + assertEquals(MimeTypeUtils.TEXT_PLAIN, + contentTypeResolver.resolve(converted.getHeaders())); + assertEquals(MimeTypeUtils.APPLICATION_JSON, + converted.getHeaders().get(XdHeaders.XD_ORIGINAL_CONTENT_TYPE)); + MessageValues reconstructed = messageBus.deserializePayloadIfNecessary(converted); + assertEquals("{\"foo\":\"foo\"}", reconstructed.getPayload()); + assertEquals(MimeTypeUtils.APPLICATION_JSON, reconstructed.get(MessageHeaders.CONTENT_TYPE)); + } + + @Test + public void testPojoSerialization() { + MessageValues convertedValues = messageBus.serializePayloadIfNecessary( + new GenericMessage(new Foo("bar")) + ); + Message converted = convertedValues.toMessage(); + MimeType mimeType = contentTypeResolver.resolve(converted.getHeaders()); + assertEquals("application", mimeType.getType()); + assertEquals("x-java-object", mimeType.getSubtype()); + assertEquals(Foo.class.getName(), mimeType.getParameter("type")); + + MessageValues reconstructed = messageBus.deserializePayloadIfNecessary(converted); + assertEquals("bar", ((Foo) reconstructed.getPayload()).getBar()); + assertNull(reconstructed.get(MessageHeaders.CONTENT_TYPE)); + } + + @Test + public void testPojoWithXJavaObjectMimeTypeNoType() { + MessageValues convertedValues = messageBus.serializePayloadIfNecessary( + new GenericMessage(new Foo("bar")) + ); + Message converted = convertedValues.toMessage(); + MimeType mimeType = contentTypeResolver.resolve(converted.getHeaders()); + assertEquals("application", mimeType.getType()); + assertEquals("x-java-object", mimeType.getSubtype()); + assertEquals(Foo.class.getName(), mimeType.getParameter("type")); + + MessageValues reconstructed = messageBus.deserializePayloadIfNecessary(converted); + assertEquals("bar", ((Foo) reconstructed.getPayload()).getBar()); + assertNull(reconstructed.get(MessageHeaders.CONTENT_TYPE)); + } + + @Test + public void testPojoWithXJavaObjectMimeTypeExplicitType() { + MessageValues convertedValues = messageBus.serializePayloadIfNecessary( + new GenericMessage(new Foo("bar")) + ); + Message converted = convertedValues.toMessage(); + MimeType mimeType = contentTypeResolver.resolve(converted.getHeaders()); + assertEquals("application", mimeType.getType()); + assertEquals("x-java-object", mimeType.getSubtype()); + assertEquals(Foo.class.getName(), mimeType.getParameter("type")); + + MessageValues reconstructed = messageBus.deserializePayloadIfNecessary(converted); + assertEquals("bar", ((Foo) reconstructed.getPayload()).getBar()); + assertNull(reconstructed.get(MessageHeaders.CONTENT_TYPE)); + } + + @Test + public void testTupleSerialization() { + Tuple payload = TupleBuilder.tuple().of("foo", "bar"); + MessageValues convertedValues = messageBus.serializePayloadIfNecessary(new GenericMessage(payload) + ); + Message converted = convertedValues.toMessage(); + MimeType mimeType = contentTypeResolver.resolve(converted.getHeaders()); + assertEquals("application", mimeType.getType()); + assertEquals("x-java-object", mimeType.getSubtype()); + assertEquals(DefaultTuple.class.getName(), mimeType.getParameter("type")); + + MessageValues reconstructed = messageBus.deserializePayloadIfNecessary(converted); + assertEquals("bar", ((Tuple) reconstructed.getPayload()).getString("foo")); + assertNull(reconstructed.get(MessageHeaders.CONTENT_TYPE)); + } + + @Test + public void mimeTypeIsSimpleObject() throws ClassNotFoundException { + MimeType mt = JavaClassMimeTypeConversion.mimeTypeFromObject(new Object()); + String className = JavaClassMimeTypeConversion.classNameFromMimeType(mt); + assertEquals(Object.class, Class.forName(className)); + } + + @Test + public void mimeTypeIsObjectArray() throws ClassNotFoundException { + MimeType mt = JavaClassMimeTypeConversion.mimeTypeFromObject(new String[0]); + String className = JavaClassMimeTypeConversion.classNameFromMimeType(mt); + assertEquals(String[].class, Class.forName(className)); + } + + @Test + public void mimeTypeIsMultiDimensionalObjectArray() throws ClassNotFoundException { + MimeType mt = JavaClassMimeTypeConversion.mimeTypeFromObject(new String[0][0][0]); + String className = JavaClassMimeTypeConversion.classNameFromMimeType(mt); + assertEquals(String[][][].class, Class.forName(className)); + } + + @Test + public void mimeTypeIsPrimitiveArray() throws ClassNotFoundException { + MimeType mt = JavaClassMimeTypeConversion.mimeTypeFromObject(new int[0]); + String className = JavaClassMimeTypeConversion.classNameFromMimeType(mt); + assertEquals(int[].class, Class.forName(className)); + } + + @Test + public void mimeTypeIsMultiDimensionalPrimitiveArray() throws ClassNotFoundException { + MimeType mt = JavaClassMimeTypeConversion.mimeTypeFromObject(new int[0][0][0]); + String className = JavaClassMimeTypeConversion.classNameFromMimeType(mt); + assertEquals(int[][][].class, Class.forName(className)); + } + + public static class Foo { + + private String bar; + + public Foo() { + } + + public Foo(String bar) { + this.bar = bar; + } + + public String getBar() { + return bar; + } + + public void setBar(String bar) { + this.bar = bar; + } + + } + + public static class Bar { + + private String foo; + + public Bar() { + } + + public Bar(String foo) { + this.foo = foo; + } + + public String getFoo() { + return foo; + } + + public void setFoo(String foo) { + this.foo = foo; + } + + } + + public class TestMessageBus extends MessageBusSupport { + + @Override + public void bindConsumer(String name, MessageChannel channel, Properties properties) { + } + + @Override + public void bindPubSubConsumer(String name, MessageChannel moduleInputChannel, + Properties properties) { + } + + @Override + public void bindPubSubProducer(String name, MessageChannel moduleOutputChannel, + Properties properties) { + } + + @Override + public void bindProducer(String name, MessageChannel channel, Properties properties) { + } + + @Override + public void bindRequestor(String name, MessageChannel requests, MessageChannel replies, + Properties properties) { + } + + @Override + public void bindReplier(String name, MessageChannel requests, MessageChannel replies, + Properties properties) { + } + } + +} diff --git a/spring-cloud-streams-codec/pom.xml b/spring-cloud-streams-codec/pom.xml index e794865ab..534d38577 100644 --- a/spring-cloud-streams-codec/pom.xml +++ b/spring-cloud-streams-codec/pom.xml @@ -3,11 +3,8 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - org.springframework.cloud spring-cloud-streams-codec - 1.0.0.BUILD-SNAPSHOT jar - spring-cloud-streams-codec Serialization library used by transport @@ -19,7 +16,6 @@ UTF-8 - 1.8 diff --git a/spring-cloud-streams-common/pom.xml b/spring-cloud-streams-common/pom.xml index d8084132a..789678193 100644 --- a/spring-cloud-streams-common/pom.xml +++ b/spring-cloud-streams-common/pom.xml @@ -3,11 +3,8 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - org.springframework.cloud spring-cloud-streams-common - 1.0.0.BUILD-SNAPSHOT jar - spring-cloud-streams-common Spring Cloud Streams common components @@ -19,7 +16,6 @@ UTF-8 - 1.8 diff --git a/spring-cloud-streams-samples/double/pom.xml b/spring-cloud-streams-samples/double/pom.xml index 4503b3861..de819bf72 100644 --- a/spring-cloud-streams-samples/double/pom.xml +++ b/spring-cloud-streams-samples/double/pom.xml @@ -3,9 +3,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - org.springframework.cloud spring-cloud-streams-sample-double - 1.0.0.BUILD-SNAPSHOT jar spring-cloud-streams-sample-double @@ -29,8 +27,8 @@ spring-cloud-streams - org.springframework.xd - spring-xd-messagebus-redis + org.springframework.cloud + spring-cloud-streams-binding-redis org.springframework.boot diff --git a/spring-cloud-streams-samples/extended/pom.xml b/spring-cloud-streams-samples/extended/pom.xml index e9368ba4d..69c2f4346 100644 --- a/spring-cloud-streams-samples/extended/pom.xml +++ b/spring-cloud-streams-samples/extended/pom.xml @@ -3,11 +3,8 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - org.springframework.cloud spring-cloud-streams-sample-extended - 1.0.0.BUILD-SNAPSHOT jar - spring-cloud-streams-sample-extended Demo project for Spring XD module @@ -41,8 +38,8 @@ spring-cloud-streams-sample-sink - org.springframework.xd - spring-xd-messagebus-redis + org.springframework.cloud + spring-cloud-streams-binding-redis org.springframework.boot diff --git a/spring-cloud-streams-samples/pom.xml b/spring-cloud-streams-samples/pom.xml index 08da4e3d0..6808cc21e 100644 --- a/spring-cloud-streams-samples/pom.xml +++ b/spring-cloud-streams-samples/pom.xml @@ -1,9 +1,8 @@ 4.0.0 - org.springframework.cloud + spring-cloud-streams-samples - 1.0.0.BUILD-SNAPSHOT pom http://projects.spring.io/spring-xd/ @@ -28,17 +27,17 @@ org.springframework.cloud spring-cloud-streams-sample-source - 1.0.0.BUILD-SNAPSHOT + ${project.version} org.springframework.cloud spring-cloud-streams-sample-sink - 1.0.0.BUILD-SNAPSHOT + ${project.version} org.springframework.cloud spring-cloud-streams-sample-transform - 1.0.0.BUILD-SNAPSHOT + ${project.version} diff --git a/spring-cloud-streams-samples/sink/pom.xml b/spring-cloud-streams-samples/sink/pom.xml index aea453056..51e370342 100644 --- a/spring-cloud-streams-samples/sink/pom.xml +++ b/spring-cloud-streams-samples/sink/pom.xml @@ -3,11 +3,8 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - org.springframework.cloud spring-cloud-streams-sample-sink - 1.0.0.BUILD-SNAPSHOT jar - spring-cloud-streams-sample-sink Demo project for Spring XD module @@ -29,8 +26,8 @@ spring-cloud-streams - org.springframework.xd - spring-xd-messagebus-redis + org.springframework.cloud + spring-cloud-streams-binding-redis org.springframework.boot diff --git a/spring-cloud-streams-samples/source/pom.xml b/spring-cloud-streams-samples/source/pom.xml index af4e741d8..2a6492683 100644 --- a/spring-cloud-streams-samples/source/pom.xml +++ b/spring-cloud-streams-samples/source/pom.xml @@ -3,11 +3,8 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - org.springframework.cloud spring-cloud-streams-sample-source - 1.0.0.BUILD-SNAPSHOT jar - spring-cloud-streams-sample-source Demo project for Spring XD module @@ -29,8 +26,8 @@ spring-cloud-streams - org.springframework.xd - spring-xd-messagebus-redis + org.springframework.cloud + spring-cloud-streams-binding-redis org.springframework.boot diff --git a/spring-cloud-streams-samples/tap/pom.xml b/spring-cloud-streams-samples/tap/pom.xml index 4c2478348..b4d397043 100644 --- a/spring-cloud-streams-samples/tap/pom.xml +++ b/spring-cloud-streams-samples/tap/pom.xml @@ -3,11 +3,8 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - org.springframework.cloud spring-cloud-streams-sample-tap - 1.0.0.BUILD-SNAPSHOT jar - spring-cloud-streams-sample-tap Demo project for Spring XD module @@ -29,8 +26,8 @@ spring-xd-runner - org.springframework.xd - spring-xd-messagebus-redis + org.springframework.cloud + spring-cloud-streams-binding-redis org.springframework.boot diff --git a/spring-cloud-streams-samples/transform/pom.xml b/spring-cloud-streams-samples/transform/pom.xml index bcf4215cd..b393a54a6 100644 --- a/spring-cloud-streams-samples/transform/pom.xml +++ b/spring-cloud-streams-samples/transform/pom.xml @@ -3,9 +3,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - org.springframework.cloud spring-cloud-streams-sample-transform - 1.0.0.BUILD-SNAPSHOT jar spring-cloud-streams-sample-transform @@ -29,8 +27,8 @@ spring-cloud-streams - org.springframework.xd - spring-xd-messagebus-redis + org.springframework.cloud + spring-cloud-streams-binding-redis org.springframework.boot diff --git a/spring-cloud-streams/pom.xml b/spring-cloud-streams/pom.xml index dd1d95264..daca63488 100644 --- a/spring-cloud-streams/pom.xml +++ b/spring-cloud-streams/pom.xml @@ -3,11 +3,8 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - org.springframework.cloud spring-cloud-streams - 1.0.0.BUILD-SNAPSHOT jar - spring-cloud-streams Messaging Microservices with Spring Integration @@ -19,7 +16,6 @@ UTF-8 - 1.8 @@ -53,20 +49,17 @@ - org.springframework.xd - spring-xd-messagebus-local + org.springframework.cloud + spring-cloud-streams-binding-local - - - org.springframework.xd - spring-xd-messagebus-redis - true + org.springframework.cloud + spring-cloud-streams-binding-redis - org.springframework.xd - spring-xd-messagebus-rabbit + org.springframework.cloud + spring-cloud-streams-binding-rabbit true diff --git a/spring-cloud-streams/src/main/java/org/springframework/cloud/streams/config/ChannelBindingAdapterConfiguration.java b/spring-cloud-streams/src/main/java/org/springframework/cloud/streams/config/ChannelBindingAdapterConfiguration.java index 70d0d2234..6d96386fb 100644 --- a/spring-cloud-streams/src/main/java/org/springframework/cloud/streams/config/ChannelBindingAdapterConfiguration.java +++ b/spring-cloud-streams/src/main/java/org/springframework/cloud/streams/config/ChannelBindingAdapterConfiguration.java @@ -54,7 +54,6 @@ import org.springframework.xd.dirt.integration.bus.serializer.kryo.FileKryoRegis import org.springframework.xd.dirt.integration.bus.serializer.kryo.KryoRegistrar; import org.springframework.xd.dirt.integration.bus.serializer.kryo.PojoCodec; - /** * @author Dave Syer * @author David Turanski @@ -171,7 +170,14 @@ public class ChannelBindingAdapterConfiguration { } - @Configuration + @ConditionalOnMissingBean(ChannelBindingProperties.class) + protected static class ModulePropertiesConfiguration { + @Bean(name = "spring.cloud.channels.CONFIGURATION_PROPERTIES") + public ChannelBindingProperties moduleProperties() { + return new ChannelBindingProperties(); + } + } + protected static class CodecConfiguration { @Autowired ApplicationContext applicationContext; @@ -179,7 +185,7 @@ public class ChannelBindingAdapterConfiguration { @Bean @ConditionalOnMissingBean(name = "codec") public MultiTypeCodec codec() { - Map kryoRegistrarMap = this.applicationContext.getBeansOfType(KryoRegistrar + Map kryoRegistrarMap = applicationContext.getBeansOfType(KryoRegistrar .class); return new PojoCodec(new ArrayList<>(kryoRegistrarMap.values())); } @@ -189,5 +195,4 @@ public class ChannelBindingAdapterConfiguration { return new FileKryoRegistrar(); } } - } diff --git a/spring-cloud-streams/src/main/java/org/springframework/cloud/streams/config/ModulePostProcessor.java b/spring-cloud-streams/src/main/java/org/springframework/cloud/streams/config/ModulePostProcessor.java index b6fba4e32..8a9a00751 100644 --- a/spring-cloud-streams/src/main/java/org/springframework/cloud/streams/config/ModulePostProcessor.java +++ b/spring-cloud-streams/src/main/java/org/springframework/cloud/streams/config/ModulePostProcessor.java @@ -47,7 +47,7 @@ public class ModulePostProcessor implements BeanDefinitionRegistryPostProcessor, } @Override - public void postProcessBeanDefinitionRegistry(BeanDefinitionRegistry registry) throws BeansException { + public void postProcessBeanDefinitionRegistry(final BeanDefinitionRegistry registry) throws BeansException { String[] beanDefinitionNames = registry.getBeanDefinitionNames(); for (String beanDefinitionName : beanDefinitionNames) { BeanDefinition beanDefinition = registry.getBeanDefinition(beanDefinitionName); @@ -77,7 +77,7 @@ public class ModulePostProcessor implements BeanDefinitionRegistryPostProcessor, } @Override - public Object postProcessBeforeInitialization(Object bean, String beanName) throws BeansException { + public Object postProcessBeforeInitialization(final Object bean, String beanName) throws BeansException { if (AnnotationUtils.findAnnotation(bean.getClass(), EnableModule.class) != null) { ReflectionUtils.doWithFields(bean.getClass(), new ReflectionUtils.FieldCallback() { @Override diff --git a/spring-cloud-streams/src/test/java/org/springframework/xd/dirt/integration/bus/MessageBusAwareChannelResolverTests.java b/spring-cloud-streams/src/test/java/org/springframework/xd/dirt/integration/bus/MessageBusAwareChannelResolverTests.java new file mode 100644 index 000000000..7dbbaf958 --- /dev/null +++ b/spring-cloud-streams/src/test/java/org/springframework/xd/dirt/integration/bus/MessageBusAwareChannelResolverTests.java @@ -0,0 +1,170 @@ +/* + * Copyright 2013-2014 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.xd.dirt.integration.bus; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; +import static org.mockito.Mockito.doReturn; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.verify; + +import java.util.ArrayList; +import java.util.List; +import java.util.Properties; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; + +import org.junit.Before; +import org.junit.Test; + +import org.springframework.beans.factory.BeanFactory; +import org.springframework.beans.factory.support.DefaultListableBeanFactory; +import org.springframework.context.support.StaticApplicationContext; +import org.springframework.integration.channel.DirectChannel; +import org.springframework.integration.channel.PublishSubscribeChannel; +import org.springframework.integration.scheduling.PollerMetadata; +import org.springframework.integration.support.DefaultMessageBuilderFactory; +import org.springframework.integration.support.MessageBuilder; +import org.springframework.integration.support.utils.IntegrationUtils; +import org.springframework.messaging.Message; +import org.springframework.messaging.MessageChannel; +import org.springframework.messaging.MessageHandler; +import org.springframework.messaging.MessagingException; +import org.springframework.scheduling.concurrent.ThreadPoolTaskScheduler; +import org.springframework.scheduling.support.PeriodicTrigger; +import org.springframework.xd.dirt.integration.bus.local.LocalMessageBus; + +/** + * @author Mark Fisher + * @author Gary Russell + */ +public class MessageBusAwareChannelResolverTests { + + private final StaticApplicationContext context = new StaticApplicationContext(); + + private volatile MessageBusAwareChannelResolver resolver; + + private volatile LocalMessageBus bus; + + @Before + public void setupContext() throws Exception { + this.bus = new LocalMessageBus(); + this.bus.setApplicationContext(context); + this.bus.afterPropertiesSet(); + this.resolver = new MessageBusAwareChannelResolver(this.bus, null); + this.resolver.setBeanFactory(context); + context.getBeanFactory().registerSingleton("channelResolver", + this.resolver); + context.registerSingleton("other", DirectChannel.class); + context.registerSingleton("taskScheduler", ThreadPoolTaskScheduler.class); + context.registerSingleton(IntegrationUtils.INTEGRATION_MESSAGE_BUILDER_FACTORY_BEAN_NAME, + DefaultMessageBuilderFactory.class); + context.refresh(); + + PollerMetadata poller = new PollerMetadata(); + poller.setTrigger(new PeriodicTrigger(1000)); + bus.setPoller(poller); + } + + @Test + public void resolveQueueChannel() { + MessageChannel registered = resolver.resolveDestination("queue:foo"); + DirectChannel testChannel = new DirectChannel(); + final CountDownLatch latch = new CountDownLatch(1); + final List> received = new ArrayList>(); + testChannel.subscribe(new MessageHandler() { + + @Override + public void handleMessage(Message message) throws MessagingException { + received.add(message); + latch.countDown(); + } + }); + bus.bindConsumer("queue:foo", testChannel, null); + assertEquals(0, received.size()); + registered.send(MessageBuilder.withPayload("hello").build()); + try { + assertTrue("latch timed out", latch.await(1, TimeUnit.SECONDS)); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + fail("interrupted while awaiting latch"); + } + assertEquals(1, received.size()); + assertEquals("hello", received.get(0).getPayload()); + context.close(); + } + + @Test + public void resolveTopicChannel() { + MessageChannel registered = resolver.resolveDestination("topic:bar"); + PublishSubscribeChannel[] testChannels = { + new PublishSubscribeChannel(), new PublishSubscribeChannel(), new PublishSubscribeChannel() + }; + final CountDownLatch latch = new CountDownLatch(testChannels.length); + final List> received = new ArrayList>(); + for (PublishSubscribeChannel testChannel : testChannels) { + testChannel.subscribe(new MessageHandler() { + + @Override + public void handleMessage(Message message) throws MessagingException { + received.add(message); + latch.countDown(); + } + }); + bus.bindPubSubConsumer("topic:bar", testChannel, null); + } + assertEquals(0, received.size()); + registered.send(MessageBuilder.withPayload("hello").build()); + try { + assertTrue("latch timed out", latch.await(1, TimeUnit.SECONDS)); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + fail("interrupted while awaiting latch"); + } + assertEquals(3, received.size()); + assertEquals("hello", received.get(0).getPayload()); + assertEquals("hello", received.get(1).getPayload()); + assertEquals("hello", received.get(2).getPayload()); + context.close(); + } + + @Test + public void resolveNonRegisteredChannel() { + MessageChannel other = resolver.resolveDestination("other"); + assertSame(context.getBean("other"), other); + } + + @Test + public void propertyPassthrough() { + Properties properties = new Properties(); + MessageBus bus = mock(MessageBus.class); + doReturn(new DirectChannel()).when(bus).bindDynamicProducer("queue:foo", properties); + doReturn(new DirectChannel()).when(bus).bindDynamicPubSubProducer("topic:bar", properties); + MessageBusAwareChannelResolver resolver = new MessageBusAwareChannelResolver(bus, properties); + BeanFactory beanFactory = new DefaultListableBeanFactory(); + resolver.setBeanFactory(beanFactory); + resolver.resolveDestination("queue:foo"); + resolver.resolveDestination("topic:bar"); + verify(bus).bindDynamicProducer("queue:foo", properties); + verify(bus).bindDynamicPubSubProducer("topic:bar", properties); + } + +} diff --git a/spring-xd-runner/pom.xml b/spring-xd-runner/pom.xml index 0b7285565..f32686558 100644 --- a/spring-xd-runner/pom.xml +++ b/spring-xd-runner/pom.xml @@ -3,11 +3,8 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - org.springframework.cloud spring-xd-runner - 1.0.0.BUILD-SNAPSHOT jar - spring-xd-runner Demo project for Spring XD Modules as apps @@ -19,7 +16,6 @@ UTF-8 - 1.8