SpringCloud/spring-cloud-stream
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Durability Configuration and Default Groups

Browse Source 
Resolves #317

Remove the `durable` binder configuration property
Make subscriber groups durable by default
Introduce `requiredGroups` property
Kafka groups (non-anonymous) now start by default at EARLIEST, which is more appropriate for new stream consumers

Addressing PR comments
This commit is contained in:
Marius Bogoevici
2016-02-18 10:59:52 -05:00
committed by Gary Russell
parent 1b1ac1c07b
commit 2fa9cda89c
13 changed files with 205 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
spring-cloud-stream-binders/spring-cloud-stream-binder-rabbit/src/main/java/org/springframework/cloud/stream/binder/rabbit/RabbitMessageChannelBinder.java
View File

@@ -134,7 +134,6 @@ public class RabbitMessageChannelBinder extends AbstractBinder<MessageChannel> {
private static final String DEAD_LETTER_EXCHANGE = "DLX";
private static final Set<Object> RABBIT_CONSUMER_PROPERTIES = new HashSet<Object>(Arrays.asList(new String[] {
BinderPropertyKeys.MAX_CONCURRENCY,
RabbitPropertiesAccessor.ACK_MODE,
RabbitPropertiesAccessor.PREFETCH,
@@ -144,7 +143,8 @@ public class RabbitMessageChannelBinder extends AbstractBinder<MessageChannel> {
RabbitPropertiesAccessor.TRANSACTED,
RabbitPropertiesAccessor.TX_SIZE,
RabbitPropertiesAccessor.AUTO_BIND_DLQ,
RabbitPropertiesAccessor.REPUBLISH_TO_DLQ
RabbitPropertiesAccessor.REPUBLISH_TO_DLQ,
RabbitPropertiesAccessor.DURABLE
}));
/**
@@ -161,7 +161,6 @@ public class RabbitMessageChannelBinder extends AbstractBinder<MessageChannel> {
*/
private static final Set<Object> SUPPORTED_CONSUMER_PROPERTIES = new SetBuilder()
.addAll(SUPPORTED_BASIC_CONSUMER_PROPERTIES)
.add(BinderPropertyKeys.DURABLE)
.add(BinderPropertyKeys.CONCURRENCY)
.add(BinderPropertyKeys.PARTITION_INDEX)
.build();
@@ -175,6 +174,7 @@ public class RabbitMessageChannelBinder extends AbstractBinder<MessageChannel> {
.add(RabbitPropertiesAccessor.PREFIX)
.add(RabbitPropertiesAccessor.REQUEST_HEADER_PATTERNS)
.add(BinderPropertyKeys.COMPRESS)
.add(BinderPropertyKeys.REQUIRED_GROUPS)
.build();
/**
@@ -232,6 +232,8 @@ public class RabbitMessageChannelBinder extends AbstractBinder<MessageChannel> {
private volatile int defaultTxSize = DEFAULT_TX_SIZE;
protected volatile boolean defaultDurableSubscription = true;
private volatile String defaultPrefix = DEFAULT_RABBIT_PREFIX;
private volatile String[] defaultRequestHeaderPatterns = DEFAULT_REQUEST_HEADER_PATTERNS;
@@ -325,6 +327,15 @@ public class RabbitMessageChannelBinder extends AbstractBinder<MessageChannel> {
this.defaultTxSize = defaultTxSize;
}
/**
* Set whether subscriptions are durable.
* @param defaultDurableSubscription true for durable (default false).
*/
public void setDefaultDurableSubscription(boolean defaultDurableSubscription) {
this.defaultDurableSubscription = defaultDurableSubscription;
}
public void setDefaultPrefix(String defaultPrefix) {
Assert.notNull(defaultPrefix, "'defaultPrefix' cannot be null");
this.defaultPrefix = defaultPrefix.trim();
@@ -555,26 +566,32 @@ public class RabbitMessageChannelBinder extends AbstractBinder<MessageChannel> {
declareExchange(exchangeName, exchange);
AmqpOutboundEndpoint endpoint = new AmqpOutboundEndpoint(rabbitTemplate);
endpoint.setExchangeName(exchange.getName());
String baseQueueName = exchangeName + ".default";
if (partitionKeyExpression == null && !StringUtils.hasText(partitionKeyExtractorClass)) {
Queue queue = new Queue(baseQueueName, true, false, false, queueArgs(properties, baseQueueName));
declareQueue(baseQueueName, queue);
autoBindDLQ(baseQueueName, baseQueueName, properties);
endpoint.setRoutingKey(name);
org.springframework.amqp.core.Binding binding = BindingBuilder.bind(queue).to(exchange).with(name);
declareBinding(baseQueueName, binding);
}
else {
endpoint.setExpressionRoutingKey(EXPRESSION_PARSER.parseExpression(buildPartitionRoutingExpression(name)));
// if the stream is partitioned, create one queue for each target partition for the default group
for (int i = 0; i < properties.getNextModuleCount(); i++) {
String partitionSuffix = "-" + i;
String partitionQueueName = baseQueueName + partitionSuffix;
Queue queue = new Queue(partitionQueueName, true, false, false,
queueArgs(properties, partitionQueueName));
declareQueue(queue.getName(), queue);
autoBindDLQ(baseQueueName, baseQueueName + partitionSuffix, properties);
declareBinding(queue.getName(), BindingBuilder.bind(queue).to(exchange).with(name + partitionSuffix));
}
for (String requiredGroupName : properties.getRequiredGroups(defaultRequiredGroups)) {
String baseQueueName = exchangeName + "." + requiredGroupName;
if (partitionKeyExpression == null && !StringUtils.hasText(partitionKeyExtractorClass)) {
Queue queue = new Queue(baseQueueName, true, false, false, queueArgs(properties, baseQueueName));
declareQueue(baseQueueName, queue);
autoBindDLQ(baseQueueName, baseQueueName, properties);
org.springframework.amqp.core.Binding binding = BindingBuilder.bind(queue).to(exchange).with(name);
declareBinding(baseQueueName, binding);
}
else {
// if the stream is partitioned, create one queue for each target partition for the default group
for (int i = 0; i < properties.getNextModuleCount(); i++) {
String partitionSuffix = "-" + i;
String partitionQueueName = baseQueueName + partitionSuffix;
Queue queue = new Queue(partitionQueueName, true, false, false,
queueArgs(properties, partitionQueueName));
declareQueue(queue.getName(), queue);
autoBindDLQ(baseQueueName, baseQueueName + partitionSuffix, properties);
declareBinding(queue.getName(), BindingBuilder.bind(queue).to(exchange).with(name + partitionSuffix));
}
}
}
configureOutboundHandler(endpoint, properties);
@@ -907,6 +924,11 @@ public class RabbitMessageChannelBinder extends AbstractBinder<MessageChannel> {
*/
private static final String REPUBLISH_TO_DLQ = "republishToDLQ";
/**
* Durable pub/sub consumer.
*/
public static final String DURABLE = "durableSubscription";
public RabbitPropertiesAccessor(Properties properties) {
super(properties);
}
@@ -967,6 +989,10 @@ public class RabbitMessageChannelBinder extends AbstractBinder<MessageChannel> {
return getProperty(REPUBLISH_TO_DLQ, defaultValue);
}
public boolean isDurable(boolean defaultValue) {
return getProperty(DURABLE, defaultValue);
}
}
}

2
spring-cloud-stream-binders/spring-cloud-stream-binder-rabbit/src/main/java/org/springframework/cloud/stream/binder/rabbit/config/RabbitBinderConfigurationProperties.java
View File

@@ -72,7 +72,7 @@ class RabbitBinderConfigurationProperties {
private int compressionLevel;
private boolean durableSubscription;
private boolean durableSubscription = true;
public AcknowledgeMode getAcknowledgeMode() {
return acknowledgeMode;

1
spring-cloud-stream-binders/spring-cloud-stream-binder-rabbit/src/main/java/org/springframework/cloud/stream/binder/rabbit/config/RabbitMessageChannelBinderConfiguration.java
View File

@@ -78,6 +78,7 @@ public class RabbitMessageChannelBinderConfiguration {
binder.setUsername(springRabbitMQProperties.getUsername());
binder.setUseSSL(springRabbitMQProperties.isUseSSL());
binder.setVhost(springRabbitMQProperties.getVhost());
binder.setDefaultDurableSubscription(rabbitBinderConfigurationProperties.isDurableSubscription());
return binder;
}

5
spring-cloud-stream-binders/spring-cloud-stream-binder-rabbit/src/test/java/org/springframework/cloud/stream/binder/rabbit/RabbitBinderTests.java
View File

@@ -340,7 +340,6 @@ public class RabbitBinderTests extends PartitionCapableBinderTests {
properties.put("maxAttempts", "1"); // disable retry
properties.put("requeue", "false");
properties.put("partitionIndex", "0");
properties.put("durableSubscription","true");
DirectChannel input0 = new DirectChannel();
input0.setBeanName("test.input0DLQ");
Binding<MessageChannel> input0Binding = binder.bindConsumer("partDLQ.0", "dlqPartGrp", input0, properties);
@@ -424,6 +423,7 @@ public class RabbitBinderTests extends PartitionCapableBinderTests {
properties.put("prefix", "bindertest.");
properties.put("autoBindDLQ", "true");
properties.put("requiredGroups", "dlqPartGrp");
properties.put("partitionKeyExtractorClass", "org.springframework.cloud.stream.binder.PartitionTestSupport");
properties.put("partitionSelectorClass", "org.springframework.cloud.stream.binder.PartitionTestSupport");
properties.put(BinderPropertyKeys.NEXT_MODULE_COUNT, "2");
@@ -437,7 +437,6 @@ public class RabbitBinderTests extends PartitionCapableBinderTests {
properties.put("maxAttempts", "1"); // disable retry
properties.put("requeue", "false");
properties.put("partitionIndex", "0");
properties.put(BinderPropertyKeys.DURABLE,"true");
DirectChannel input0 = new DirectChannel();
input0.setBeanName("test.input0DLQ");
Binding<MessageChannel> input0Binding = binder.bindConsumer("partDLQ.1", "dlqPartGrp", input0, properties);
@@ -563,6 +562,7 @@ public class RabbitBinderTests extends PartitionCapableBinderTests {
properties.put("batchBufferLimit", "100000");
properties.put("batchTimeout", "30000");
properties.put("compress", "true");
properties.put("requiredGroups", "default");
DirectChannel output = new DirectChannel();
output.setBeanName("batchingProducer");
@@ -657,6 +657,7 @@ public class RabbitBinderTests extends PartitionCapableBinderTests {
MessageChannel outputChannel = new DirectChannel();
Binding<MessageChannel> pubSubProducerBinding = binder.bindProducer("latePubSub", outputChannel, properties);
QueueChannel pubSubInputChannel = new QueueChannel();
properties.setProperty("durableSubscription", "false");
Binding<MessageChannel> nonDurableConsumerBinding = binder.bindConsumer("latePubSub", "lategroup", pubSubInputChannel, properties);
QueueChannel durablePubSubInputChannel = new QueueChannel();
properties.setProperty("durableSubscription", "true");