INT-3325 Add Redis Channel Message Group Store

JIRA: https://jira.spring.io/browse/INT-3325
JIRA: https://jira.spring.io/browse/INT-1870

Optimized MGS for QueueChannel - uses a LIST for
each channel and LPUSH, RPOP.

* Also fix MutableMessage to be Serializable

INT-1870 Priority Redis Channel Message Store

Supports priorities 0-9 (+ no priority).

Priorities out of that range are treated as no priority.

Polishing - Add Marker Interfaces

* Emit a `WARN` log if a channel is used with a regular MessageGroupStore
* Allow message-store on namespace when defining a priority channel

INT-3325 Polishing; PR Comments

Fix some typos in JavaDocs and Docs
This commit is contained in:
Gary Russell
2014-03-18 12:33:41 +02:00
committed by Artem Bilan
parent a9faa5836f
commit a8c8a4fed5
20 changed files with 846 additions and 70 deletions

View File

@@ -0,0 +1,135 @@
/*
* Copyright 2014 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.redis.store;
import java.util.List;
import java.util.Set;
import org.springframework.beans.factory.BeanNameAware;
import org.springframework.beans.factory.InitializingBean;
import org.springframework.data.redis.connection.RedisConnectionFactory;
import org.springframework.data.redis.core.RedisTemplate;
import org.springframework.data.redis.serializer.JdkSerializationRedisSerializer;
import org.springframework.data.redis.serializer.RedisSerializer;
import org.springframework.data.redis.serializer.StringRedisSerializer;
import org.springframework.integration.store.ChannelMessageStore;
import org.springframework.integration.store.MessageGroup;
import org.springframework.integration.store.SimpleMessageGroup;
import org.springframework.jmx.export.annotation.ManagedAttribute;
import org.springframework.messaging.Message;
import org.springframework.util.Assert;
/**
* Specialized Redis {@link ChannelMessageStore} that uses a list to back a QueueChannel.
* <p>
* Requires {@link #setBeanName(String)} which is used as part of the key.
*
* @author Gary Russell
* @since 4.0
*
*/
public class RedisChannelMessageStore implements ChannelMessageStore, BeanNameAware, InitializingBean {
private final RedisTemplate<Object, Message<?>> redisTemplate;
private String beanName;
/**
* Construct a message store that uses Java Serialization for messages.
*
* @param connectionFactory The redis connection factory.
*/
public RedisChannelMessageStore(RedisConnectionFactory connectionFactory) {
this.redisTemplate = new RedisTemplate<Object, Message<?>>();
this.redisTemplate.setConnectionFactory(connectionFactory);
this.redisTemplate.setKeySerializer(new StringRedisSerializer());
this.redisTemplate.setValueSerializer(new JdkSerializationRedisSerializer());
this.redisTemplate.afterPropertiesSet();
}
/**
* Use a different serializer (default {@link JdkSerializationRedisSerializer} for
* the {@link Message}.
*
* @param valueSerializer The value serializer.
*/
public void setValueSerializer(RedisSerializer<?> valueSerializer) {
Assert.notNull(valueSerializer, "'valueSerializer' must not be null");
this.redisTemplate.setValueSerializer(valueSerializer);
}
@Override
public void setBeanName(String name) {
Assert.notNull(name, "'beanName' must not be null");
this.beanName = name;
}
protected String getBeanName() {
return beanName;
}
protected RedisTemplate<Object, Message<?>> getRedisTemplate() {
return redisTemplate;
}
@Override
public void afterPropertiesSet() throws Exception {
Assert.notNull(this.beanName, "'beanName' must not be null");
}
@Override
@ManagedAttribute
public int messageGroupSize(Object groupId) {
return (int) this.redisTemplate.boundListOps(groupId).size().longValue();
}
@Override
public MessageGroup getMessageGroup(Object groupId) {
List<Message<?>> messages = this.redisTemplate.boundListOps(groupId).range(0, -1);
return new SimpleMessageGroup(messages, groupId);
}
@Override
public MessageGroup addMessageToGroup(Object groupId, Message<?> message) {
this.redisTemplate.boundListOps(groupId).leftPush(message);
return null;
}
public void removeMessageGroup(Object groupId) {
this.redisTemplate.boundListOps(groupId).trim(1, 0);
}
@Override
public Message<?> pollMessageFromGroup(Object groupId) {
return this.redisTemplate.boundListOps(groupId).rightPop();
}
@ManagedAttribute
public int getMessageCountForAllMessageGroups() {
Set<?> keys = this.redisTemplate.keys(this.beanName + ":*");
int count = 0;
for (Object key : keys) {
count += this.messageGroupSize(key);
}
return count;
}
@ManagedAttribute
public int getMessageGroupCount() {
return this.redisTemplate.keys(this.beanName + ":*").size();
}
}

View File

@@ -0,0 +1,166 @@
/*
* Copyright 2014 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.redis.store;
import java.util.Collections;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Set;
import org.springframework.data.redis.connection.RedisConnectionFactory;
import org.springframework.integration.IntegrationMessageHeaderAccessor;
import org.springframework.integration.store.MessageGroup;
import org.springframework.integration.store.PriorityCapableChannelMessageStore;
import org.springframework.integration.store.SimpleMessageGroup;
import org.springframework.jmx.export.annotation.ManagedAttribute;
import org.springframework.messaging.Message;
import org.springframework.util.Assert;
/**
* Specialized Redis {@link PriorityCapableChannelMessageStore} that uses lists to back a QueueChannel.
* Messages are removed in priority order ({@link IntegrationMessageHeaderAccessor#PRIORITY}).
* Priorities 0-9 are supported; higher values are treated with the same priority (none)
* as messages with no priority header (retrieved after any messages that have a priority).
* <p>
* Requires that groupId is a String.
*
* @author Gary Russell
* @since 4.0
*
*/
public class RedisChannelPriorityMessageStore extends RedisChannelMessageStore implements PriorityCapableChannelMessageStore {
public RedisChannelPriorityMessageStore(RedisConnectionFactory connectionFactory) {
super(connectionFactory);
}
@Override
public boolean isPriorityEnabled() {
return true;
}
@Override
@ManagedAttribute
public int messageGroupSize(Object groupId) {
Assert.isInstanceOf(String.class, groupId);
List<String> list = sortedKeys((String) groupId);
int count = 0;
for (String key : list) {
count += this.getRedisTemplate().boundListOps(key).size();
}
return count;
}
@Override
public MessageGroup getMessageGroup(Object groupId) {
Assert.isInstanceOf(String.class, groupId);
List<Message<?>> allMessages = new LinkedList<Message<?>>();
List<String> list = sortedKeys((String) groupId);
for (String key : list) {
List<Message<?>> messages = this.getRedisTemplate().boundListOps(key).range(0, -1);
allMessages.addAll(messages);
}
return new SimpleMessageGroup(allMessages, groupId);
}
@Override
public MessageGroup addMessageToGroup(Object groupId, Message<?> message) {
Assert.isInstanceOf(String.class, groupId);
String key = (String) groupId;
Integer priority = new IntegrationMessageHeaderAccessor(message).getPriority();
if (priority != null && priority < 10 && priority >= 0) {
key = key + ":" + priority;
}
else {
key = key + ":z";
}
return super.addMessageToGroup(key, message);
}
@Override
public Message<?> pollMessageFromGroup(Object groupId) {
Assert.isInstanceOf(String.class, groupId);
List<String> list = sortedKeys((String) groupId);
Message<?> message;
for (String key : list) {
message = super.pollMessageFromGroup(key);
if (message != null) {
return message;
}
}
return null;
}
private List<String> sortedKeys(String groupId) {
Set<Object> keys = this.getRedisTemplate().keys(groupId == null ? (this.getBeanName() + ":*") : (groupId + "*"));
List<String> list = new LinkedList<String>();
for (Object key : keys) {
Assert.isInstanceOf(String.class, key);
list.add((String) key);
}
Collections.sort(list);
return list;
}
@Override
@ManagedAttribute
public int getMessageGroupCount() {
Set<Object> narrowedKeys = narrowedKeys();
return narrowedKeys.size();
}
private Set<Object> narrowedKeys() {
Set<Object> keys = this.getRedisTemplate().keys(this.getBeanName() + ":*");
Set<Object> narrowedKeys = new HashSet<Object>();
for (Object key : keys) {
Assert.isInstanceOf(String.class, key);
String keyString = (String) key;
int lastIndexOfColon = keyString.lastIndexOf(":");
if (keyString.indexOf(":") != lastIndexOfColon) {
narrowedKeys.add(keyString.substring(0, lastIndexOfColon));
}
else {
narrowedKeys.add(key);
}
}
return narrowedKeys;
}
@Override
public void removeMessageGroup(Object groupId) {
Assert.isInstanceOf(String.class, groupId);
List<String> list = sortedKeys((String) groupId);
for (String key : list) {
super.removeMessageGroup(key);
}
}
@Override
@ManagedAttribute
public int getMessageCountForAllMessageGroups() {
Set<Object> narrowedKeys = narrowedKeys();
int count = 0;
for (Object key : narrowedKeys) {
count += this.messageGroupSize(key);
}
return count;
}
}

View File

@@ -0,0 +1,40 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:int="http://www.springframework.org/schema/integration"
xsi:schemaLocation="http://www.springframework.org/schema/integration http://www.springframework.org/schema/integration/spring-integration.xsd
http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd">
<bean id="cms" class="org.springframework.integration.redis.store.RedisChannelMessageStore">
<constructor-arg>
<bean class="org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory">
<property name="port" value="#{T(org.springframework.integration.redis.rules.RedisAvailableRule).REDIS_PORT}"/>
</bean>
</constructor-arg>
</bean>
<int:channel id="testChannel1">
<int:queue message-store="cms" />
</int:channel>
<int:channel id="testChannel2">
<int:queue message-store="cms" />
</int:channel>
<bean id="priorityCms" class="org.springframework.integration.redis.store.RedisChannelPriorityMessageStore">
<constructor-arg>
<bean class="org.springframework.data.redis.connection.lettuce.LettuceConnectionFactory">
<property name="port" value="#{T(org.springframework.integration.redis.rules.RedisAvailableRule).REDIS_PORT}"/>
</bean>
</constructor-arg>
</bean>
<int:channel id="testChannel3">
<int:priority-queue message-store="priorityCms" />
</int:channel>
<int:channel id="testChannel4">
<int:priority-queue message-store="priorityCms" />
</int:channel>
</beans>

View File

@@ -0,0 +1,175 @@
/*
* Copyright 2014 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.redis.store;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertThat;
import org.hamcrest.Matchers;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.integration.IntegrationMessageHeaderAccessor;
import org.springframework.integration.message.MutableMessage;
import org.springframework.integration.redis.rules.RedisAvailable;
import org.springframework.integration.redis.rules.RedisAvailableTests;
import org.springframework.integration.support.MessageBuilder;
import org.springframework.messaging.Message;
import org.springframework.messaging.PollableChannel;
import org.springframework.messaging.support.GenericMessage;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
/**
* @author Gary Russell
* @since 4.0
*
*/
@ContextConfiguration
@RunWith(SpringJUnit4ClassRunner.class)
public class RedisChannelMessageStoreTests extends RedisAvailableTests {
@Autowired
private PollableChannel testChannel1;
@Autowired
private PollableChannel testChannel2;
@Autowired
private PollableChannel testChannel3;
@Autowired
private PollableChannel testChannel4;
@Autowired
private RedisChannelMessageStore cms;
@Autowired
private RedisChannelMessageStore priorityCms;
@Before
public void setup() {
this.cms.removeMessageGroup("cms:testChannel1");
this.cms.removeMessageGroup("cms:testChannel2");
this.priorityCms.removeMessageGroup("priorityCms:testChannel3");
this.priorityCms.removeMessageGroup("priorityCms:testChannel4");
}
@Test
@RedisAvailable
public void testChannel() {
for (int i = 0; i < 10; i++) {
this.testChannel1.send(new GenericMessage<Integer>(i));
}
assertEquals(1, this.cms.getMessageGroupCount());
assertEquals(10, this.cms.messageGroupSize("cms:testChannel1"));
assertEquals(10, this.cms.getMessageGroup("cms:testChannel1").size());
for (int i = 0; i < 10; i++) {
this.testChannel2.send(new MutableMessage<Integer>(i));
}
assertEquals(2, this.cms.getMessageGroupCount());
assertEquals(10, this.cms.messageGroupSize("cms:testChannel2"));
assertEquals(10, this.cms.getMessageGroup("cms:testChannel2").size());
assertEquals(20, this.cms.getMessageCountForAllMessageGroups());
for (int i = 0; i < 10; i++) {
Message<?> out = this.testChannel1.receive(0);
assertThat(out, Matchers.instanceOf(GenericMessage.class));
assertEquals(Integer.valueOf(i), out.getPayload());
}
assertNull(this.testChannel1.receive(0));
for (int i = 0; i < 10; i++) {
Message<?> out = this.testChannel2.receive(0);
assertThat(out, Matchers.instanceOf(MutableMessage.class));
assertEquals(Integer.valueOf(i), out.getPayload());
}
assertNull(this.testChannel2.receive(0));
assertEquals(0, this.cms.getMessageGroupCount());
for (int i = 0; i < 10; i++) {
this.testChannel1.send(new GenericMessage<Integer>(i));
}
assertEquals(1, this.cms.getMessageGroupCount());
assertEquals(10, this.cms.messageGroupSize("cms:testChannel1"));
this.cms.removeMessageGroup("cms:testChannel1");
assertEquals(0, this.cms.getMessageGroupCount());
assertEquals(0, this.cms.messageGroupSize("cms:testChannel1"));
}
@Test
@RedisAvailable
public void testPriority() {
for (int i = 0; i < 10; i++) {
Message<Integer> message = MessageBuilder.withPayload(i).setPriority(9-i).build();
this.testChannel3.send(message);
this.testChannel3.send(message);
}
this.testChannel3.send(MessageBuilder.withPayload(99).setPriority(199).build());
this.testChannel3.send(MessageBuilder.withPayload(98).build());
assertEquals(1, this.priorityCms.getMessageGroupCount());
assertEquals(22, this.priorityCms.messageGroupSize("priorityCms:testChannel3"));
assertEquals(22, this.priorityCms.getMessageCountForAllMessageGroups());
assertEquals(22, this.priorityCms.getMessageGroup("priorityCms:testChannel3").size());
this.testChannel4.send(MessageBuilder.withPayload(98).build());
this.testChannel4.send(MessageBuilder.withPayload(99).setPriority(5).build());
assertEquals(2, this.priorityCms.getMessageGroupCount());
assertEquals(2, this.priorityCms.getMessageGroup("priorityCms:testChannel4").size());
assertEquals(2, this.priorityCms.messageGroupSize("priorityCms:testChannel4"));
assertEquals(24, this.priorityCms.getMessageCountForAllMessageGroups());
for (int i = 0; i < 10; i++) {
Message<?> m = this.testChannel3.receive(0);
assertNotNull(m);
assertEquals(Integer.valueOf(i), new IntegrationMessageHeaderAccessor(m).getPriority());
assertEquals(Integer.valueOf(9-i), m.getPayload());
m = this.testChannel3.receive(0);
assertNotNull(m);
assertEquals(Integer.valueOf(i), new IntegrationMessageHeaderAccessor(m).getPriority());
}
Message<?> m = this.testChannel3.receive(0);
assertNotNull(m);
assertEquals(Integer.valueOf(199), new IntegrationMessageHeaderAccessor(m).getPriority());
assertEquals(Integer.valueOf(99), m.getPayload());
m = this.testChannel3.receive(0);
assertNotNull(m);
assertNull(new IntegrationMessageHeaderAccessor(m).getPriority());
assertEquals(Integer.valueOf(98), m.getPayload());
assertEquals(0, this.priorityCms.messageGroupSize("priorityCms:testChannel3"));
m = this.testChannel4.receive(0);
assertNotNull(m);
assertEquals(Integer.valueOf(5), new IntegrationMessageHeaderAccessor(m).getPriority());
m = this.testChannel4.receive(0);
assertNotNull(m);
assertNull(new IntegrationMessageHeaderAccessor(m).getPriority());
assertEquals(0, this.priorityCms.getMessageGroupCount());
assertEquals(0, this.priorityCms.getMessageCountForAllMessageGroups());
assertNull(this.testChannel3.receive(0));
assertNull(this.testChannel4.receive(0));
for (int i = 0; i < 10; i++) {
this.testChannel3.send(new GenericMessage<Integer>(i));
}
assertEquals(1, this.priorityCms.getMessageGroupCount());
assertEquals(10, this.priorityCms.messageGroupSize("priorityCms:testChannel3"));
this.priorityCms.removeMessageGroup("priorityCms:testChannel3");
assertEquals(0, this.priorityCms.getMessageGroupCount());
assertEquals(0, this.priorityCms.messageGroupSize("priorityCms:testChannel3"));
}
}

View File

@@ -29,6 +29,8 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import junit.framework.AssertionFailedError;
import org.junit.Ignore;
import org.junit.Test;
@@ -46,8 +48,6 @@ import org.springframework.messaging.Message;
import org.springframework.messaging.MessageChannel;
import org.springframework.messaging.support.GenericMessage;
import junit.framework.AssertionFailedError;
/**
* @author Oleg Zhurakousky
* @author Artem Bilan
@@ -318,6 +318,7 @@ public class RedisMessageGroupStoreTests extends RedisAvailableTests {
executor = Executors.newCachedThreadPool();
executor.execute(new Runnable() {
@Override
public void run() {
MessageGroup group = store1.addMessageToGroup(1, message);
if (group.getMessages().size() != 1){
@@ -327,6 +328,7 @@ public class RedisMessageGroupStoreTests extends RedisAvailableTests {
}
});
executor.execute(new Runnable() {
@Override
public void run() {
MessageGroup group = store2.removeMessageFromGroup(1, message);
if (group.getMessages().size() != 0){
@@ -367,6 +369,7 @@ public class RedisMessageGroupStoreTests extends RedisAvailableTests {
Message<?> m3 = MessageBuilder.withPayload("3").setSequenceNumber(3).setSequenceSize(3).setCorrelationId(1).build();
input.send(m3);
assertNotNull(output.receive(1000));
context.close();
}
}