INT-3387: MessageGroupStore Improvements

JIRA: https://jira.spring.io/browse/INT-3387,
https://jira.spring.io/browse/INT-3806

* Introduce
```
MessageGroupStore

void addMessagesToGroup(Object groupId, Message<?>... messages);
```
And implement it in all stores.

* Use new `addMessagesToGroup` where it is reasonable, e.g. `DelayHandler`
* Optimize test-case to use a new store method (where it is possible)
* Fix timing delays in the `JdbcMessageStoreTests`
* Introduce `PersistentMessageGroup`
* Add `AbstractMessageGroupStore#proxyMessageGroupForLazyLoad` to wrap the raw `MessageGroup` to the `PersistentMessageGroup` for lazy-load
* Rework `MessageGroupMetadata` do not be `immutable` and allow to store/restore in the `AbstractKeyValueMessageStore` only the `MessageGroupMetadata`
* Refactor `ResequencingMessageHandler` and `SequenceSizeReleaseStrategy` a bit for better performance when interact with the `MessageGroup`
* Add `AbstractMessageGroupStore#setLazyLoadMessageGroups` to switch off the `lazy-load` behavior and restore the previous full `MessageGroup` logic
* Add `What's New` note and `message-store.adoc` paragraph for the lazy-load functionality

`GroupType.PERSISTENT` and not lazy by default

PR Comments

Fix `JdbcMessageStoreTests` timing issues

Address PR comments

* Add performance test to the `ConfigurableMongoDbMessageGroupStoreTests`
* Add JavaDocs for the `MessageGroupFactory` methods
* Add `log4j.properties` into the `test` MongoDB module for better traceability
* Fix `JdbcMessageStore#getOneMessageFromGroup()` over the `doPollForMessage()` delegation.
The `jdbcTemplate.queryForObject()` requires exactly one and only one raw in `resultSet`
* Add performance test results into the `message-store.adoc`

spring-integration-mongodb/src/test/java/org/springframework/integration/mongodb/store/AbstractMongoDbMessageGroupStoreTests.java

@@ -16,10 +16,12 @@
 
 package org.springframework.integration.mongodb.store;
 
+import static org.hamcrest.Matchers.containsString;
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertNotNull;
 import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertThat;
 import static org.junit.Assert.assertTrue;
 
 import java.util.ArrayList;
@@ -42,7 +44,6 @@ import org.springframework.integration.store.AbstractBatchingMessageGroupStore;
 import org.springframework.integration.store.MessageGroup;
 import org.springframework.integration.store.MessageGroupStore;
 import org.springframework.integration.store.MessageStore;
-import org.springframework.integration.store.SimpleMessageGroup;
 import org.springframework.integration.support.MessageBuilder;
 import org.springframework.messaging.Message;
 import org.springframework.messaging.MessageChannel;
@@ -63,10 +64,11 @@ public abstract class AbstractMongoDbMessageGroupStoreTests extends MongoDbAvail
 	public void testNonExistingEmptyMessageGroup() throws Exception {
 		this.cleanupCollections(new SimpleMongoDbFactory(new MongoClient(), "test"));
 		MessageGroupStore store = getMessageGroupStore();
+		store.addMessagesToGroup(1, new GenericMessage<Object>("foo"));
 		MessageGroup messageGroup = store.getMessageGroup(1);
 		assertNotNull(messageGroup);
-		assertTrue(messageGroup instanceof SimpleMessageGroup);
-		assertEquals(0, messageGroup.size());
+		assertThat(messageGroup.getClass().getName(), containsString("PersistentMessageGroup"));
+		assertEquals(1, messageGroup.size());
 	}
 
 	@Test
@@ -78,7 +80,7 @@ public abstract class AbstractMongoDbMessageGroupStoreTests extends MongoDbAvail
 		MessageGroup messageGroup = store.getMessageGroup(1);
 		Message<?> messageA = new GenericMessage<String>("A");
 		Message<?> messageB = new GenericMessage<String>("B");
-		store.addMessageToGroup(1, messageA);
+		store.addMessagesToGroup(1, messageA);
 		messageGroup = store.addMessageToGroup(1, messageB);
 		assertNotNull(messageGroup);
 		assertEquals(2, messageGroup.size());
@@ -101,7 +103,7 @@ public abstract class AbstractMongoDbMessageGroupStoreTests extends MongoDbAvail
 		Message<?> messageA = MessageBuilder.withPayload("A").setHeader("foo", uuidA).build();
 		UUID uuidB = UUID.randomUUID();
 		Message<?> messageB = MessageBuilder.withPayload("B").setHeader("foo", uuidB).build();
-		store.addMessageToGroup(id, messageA);
+		store.addMessagesToGroup(id, messageA);
 		messageGroup = store.addMessageToGroup(id, messageB);
 		assertNotNull(messageGroup);
 		assertEquals(2, messageGroup.size());
@@ -122,8 +124,7 @@ public abstract class AbstractMongoDbMessageGroupStoreTests extends MongoDbAvail
 		MessageGroupStore store = this.getMessageGroupStore();
 		Message<?> messageA = new GenericMessage<String>("A");
 		Message<?> messageB = new GenericMessage<String>("B");
-		store.addMessageToGroup(1, messageA);
-		store.addMessageToGroup(1, messageB);
+		store.addMessagesToGroup(1, messageA, messageB);
 		assertEquals(2, store.messageGroupSize(1));
 	}
 
@@ -134,9 +135,9 @@ public abstract class AbstractMongoDbMessageGroupStoreTests extends MongoDbAvail
 		MessageGroupStore store = this.getMessageGroupStore();
 		Message<?> messageA = new GenericMessage<String>("A");
 		Message<?> messageB = new GenericMessage<String>("B");
-		store.addMessageToGroup(1, messageA);
+		store.addMessagesToGroup(1, messageA);
 		Thread.sleep(10);
-		store.addMessageToGroup(1, messageB);
+		store.addMessagesToGroup(1, messageB);
 		assertEquals(2, store.messageGroupSize(1));
 		Message<?> out = store.pollMessageFromGroup(1);
 		assertNotNull(out);
@@ -153,10 +154,10 @@ public abstract class AbstractMongoDbMessageGroupStoreTests extends MongoDbAvail
 		this.cleanupCollections(new SimpleMongoDbFactory(new MongoClient(), "test"));
 		MessageGroupStore store = this.getMessageGroupStore();
 		Message<?> messageA = new GenericMessage<String>("A");
-		store.addMessageToGroup(1, messageA);
-		store.addMessageToGroup(2, messageA);
-		store.addMessageToGroup(3, messageA);
-		store.addMessageToGroup(4, messageA);
+		store.addMessagesToGroup(1, messageA);
+		store.addMessagesToGroup(2, messageA);
+		store.addMessagesToGroup(3, messageA);
+		store.addMessagesToGroup(4, messageA);
 		assertEquals(1, store.messageGroupSize(1));
 		assertEquals(1, store.messageGroupSize(2));
 		assertEquals(1, store.messageGroupSize(3));
@@ -190,10 +191,10 @@ public abstract class AbstractMongoDbMessageGroupStoreTests extends MongoDbAvail
 		MessageGroupStore store = this.getMessageGroupStore();
 
 		Message<?> messageA = new GenericMessage<String>("A");
-		store.addMessageToGroup(1, messageA);
-		store.addMessageToGroup(2, messageA);
-		store.addMessageToGroup(3, messageA);
-		store.addMessageToGroup(4, messageA);
+		store.addMessagesToGroup(1, messageA);
+		store.addMessagesToGroup(2, messageA);
+		store.addMessagesToGroup(3, messageA);
+		store.addMessagesToGroup(4, messageA);
 		assertEquals(1, store.messageGroupSize(1));
 		assertEquals(1, store.messageGroupSize(2));
 		assertEquals(1, store.messageGroupSize(3));
@@ -258,7 +259,7 @@ public abstract class AbstractMongoDbMessageGroupStoreTests extends MongoDbAvail
 		MessageGroup messageGroup = store.getMessageGroup(1);
 		Message<?> messageA = new GenericMessage<String>("A");
 		Message<?> messageB = new GenericMessage<String>("B");
-		store.addMessageToGroup(1, messageA);
+		store.addMessagesToGroup(1, messageA);
 		messageGroup = store.addMessageToGroup(1, messageB);
 		assertNotNull(messageGroup);
 		assertEquals(2, messageGroup.size());
@@ -306,7 +307,7 @@ public abstract class AbstractMongoDbMessageGroupStoreTests extends MongoDbAvail
 		MessageGroup messageGroup = store.getMessageGroup(1);
 		assertNotNull(messageGroup);
 		Message<?> message = new GenericMessage<String>("Hello");
-		store.addMessageToGroup(messageGroup.getGroupId(), message);
+		store.addMessagesToGroup(messageGroup.getGroupId(), message);
 		store.completeGroup(messageGroup.getGroupId());
 		messageGroup = store.getMessageGroup(1);
 		assertTrue(messageGroup.isComplete());
@@ -321,7 +322,7 @@ public abstract class AbstractMongoDbMessageGroupStoreTests extends MongoDbAvail
 		MessageGroup messageGroup = store.getMessageGroup(1);
 		assertNotNull(messageGroup);
 		Message<?> message = new GenericMessage<String>("Hello");
-		store.addMessageToGroup(messageGroup.getGroupId(), message);
+		store.addMessagesToGroup(messageGroup.getGroupId(), message);
 		store.setLastReleasedSequenceNumberForGroup(messageGroup.getGroupId(), 5);
 		messageGroup = store.getMessageGroup(1);
 		assertEquals(5, messageGroup.getLastReleasedMessageSequenceNumber());
@@ -335,8 +336,7 @@ public abstract class AbstractMongoDbMessageGroupStoreTests extends MongoDbAvail
 
 		MessageGroup messageGroup = store.getMessageGroup(1);
 		Message<?> message = new GenericMessage<String>("2");
-		store.addMessageToGroup(1, new GenericMessage<String>("1"));
-		store.addMessageToGroup(1, message);
+		store.addMessagesToGroup(1, new GenericMessage<String>("1"), message);
 		messageGroup = store.addMessageToGroup(1, new GenericMessage<String>("3"));
 		assertNotNull(messageGroup);
 		assertEquals(3, messageGroup.size());
@@ -355,9 +355,7 @@ public abstract class AbstractMongoDbMessageGroupStoreTests extends MongoDbAvail
 		MessageGroupStore store2 = this.getMessageGroupStore();
 
 		Message<?> message = new GenericMessage<String>("1");
-		store1.addMessageToGroup(1, message);
-		store2.addMessageToGroup(1, new GenericMessage<String>("2"));
-		store1.addMessageToGroup(1, new GenericMessage<String>("3"));
+		store1.addMessagesToGroup(1, message, new GenericMessage<String>("2"), new GenericMessage<String>("3"));
 
 		MessageGroupStore store3 = this.getMessageGroupStore();
 
@@ -380,9 +378,9 @@ public abstract class AbstractMongoDbMessageGroupStoreTests extends MongoDbAvail
 		MessageGroupStore store2 = this.getMessageGroupStore();
 
 		Message<?> message = new GenericMessage<String>("1");
-		store2.addMessageToGroup(1, message);
-		store1.addMessageToGroup(2, new GenericMessage<String>("2"));
-		store2.addMessageToGroup(3, new GenericMessage<String>("3"));
+		store2.addMessagesToGroup(1, message);
+		store1.addMessagesToGroup(2, new GenericMessage<String>("2"));
+		store2.addMessagesToGroup(3, new GenericMessage<String>("3"));
 
 		MessageGroupStore store3 = this.getMessageGroupStore();
 		Iterator<MessageGroup> iterator = store3.iterator();
@@ -415,7 +413,7 @@ public abstract class AbstractMongoDbMessageGroupStoreTests extends MongoDbAvail
 		List<Message<?>> messages = new ArrayList<Message<?>>();
 		for (int i = 0; i < 25; i++) {
 			Message<String> message = MessageBuilder.withPayload("foo").setCorrelationId(groupId).build();
-			messageStore.addMessageToGroup(groupId, message);
+			messageStore.addMessagesToGroup(groupId, message);
 			messages.add(message);
 		}
 		MessageGroup group = messageStore.getMessageGroup(groupId);
@@ -477,20 +475,41 @@ public abstract class AbstractMongoDbMessageGroupStoreTests extends MongoDbAvail
 		MessageChannel input = context.getBean("inputChannel", MessageChannel.class);
 		QueueChannel output = context.getBean("outputChannel", QueueChannel.class);
 
-		Message<?> m1 = MessageBuilder.withPayload("1").setSequenceNumber(1).setSequenceSize(3).setCorrelationId(1).build();
-		Message<?> m2 = MessageBuilder.withPayload("2").setSequenceNumber(2).setSequenceSize(3).setCorrelationId(1).build();
+		Message<?> m1 = MessageBuilder.withPayload("1")
+				.setSequenceNumber(1)
+				.setSequenceSize(10)
+				.setCorrelationId(1)
+				.build();
+		Message<?> m2 = MessageBuilder.withPayload("2")
+				.setSequenceNumber(2)
+				.setSequenceSize(10)
+				.setCorrelationId(1)
+				.build();
 		input.send(m1);
 		assertNull(output.receive(1000));
 		input.send(m2);
 		assertNull(output.receive(1000));
+
+		for (int i = 3; i < 10; i++) {
+			input.send(MessageBuilder.withPayload("" + i)
+					.setSequenceNumber(i)
+					.setSequenceSize(10)
+					.setCorrelationId(1)
+					.build());
+		}
+
 		context.close();
 
 		context = new ClassPathXmlApplicationContext(config, this.getClass());
 		input = context.getBean("inputChannel", MessageChannel.class);
 		output = context.getBean("outputChannel", QueueChannel.class);
 
-		Message<?> m3 = MessageBuilder.withPayload("3").setSequenceNumber(3).setSequenceSize(3).setCorrelationId(1).build();
-		input.send(m3);
+		Message<?> m10 = MessageBuilder.withPayload("10")
+				.setSequenceNumber(10)
+				.setSequenceSize(10)
+				.setCorrelationId(1)
+				.build();
+		input.send(m10);
 		assertNotNull(output.receive(2000));
 		context.close();
 	}
@@ -511,7 +530,7 @@ public abstract class AbstractMongoDbMessageGroupStoreTests extends MongoDbAvail
 
 		message = MessageHistory.write(message, fooChannel);
 		message = MessageHistory.write(message, barChannel);
-		store.addMessageToGroup(1, message);
+		store.addMessagesToGroup(1, message);
 		MessageGroup group = store.getMessageGroup(1);
 		assertNotNull(group);
 		Collection<Message<?>> messages = group.getMessages();

spring-integration-mongodb/src/test/java/org/springframework/integration/mongodb/store/ConfigurableMongoDbMessageGroupStoreTests.java

@@ -23,6 +23,7 @@ import static org.junit.Assert.assertThat;
 import java.util.Map;
 
 import org.hamcrest.Matchers;
+import org.junit.Ignore;
 import org.junit.Test;
 
 import org.springframework.context.support.ClassPathXmlApplicationContext;
@@ -34,10 +35,13 @@ import org.springframework.integration.IntegrationMessageHeaderAccessor;
 import org.springframework.integration.channel.PriorityChannel;
 import org.springframework.integration.channel.QueueChannel;
 import org.springframework.integration.mongodb.rules.MongoDbAvailable;
+import org.springframework.integration.store.AbstractMessageGroupStore;
 import org.springframework.integration.store.MessageStore;
 import org.springframework.integration.support.MessageBuilder;
 import org.springframework.integration.test.util.TestUtils;
 import org.springframework.messaging.Message;
+import org.springframework.messaging.MessageChannel;
+import org.springframework.util.StopWatch;
 
 import com.mongodb.DBObject;
 import com.mongodb.MongoClient;
@@ -68,7 +72,49 @@ public class ConfigurableMongoDbMessageGroupStoreTests extends AbstractMongoDbMe
 	@Test
 	@MongoDbAvailable
 	public void testWithAggregatorWithShutdown() throws Exception {
-		super.testWithAggregatorWithShutdown("mongo-aggregator-confugurable-config.xml");
+		super.testWithAggregatorWithShutdown("mongo-aggregator-configurable-config.xml");
+	}
+
+	@Test
+	@Ignore("The performance test. Enough slow. Also needs the release strategy changed to size() == 1000")
+	@MongoDbAvailable
+	public void messageGroupStoreLazyLoadPerformance() throws Exception {
+		cleanupCollections(new SimpleMongoDbFactory(new MongoClient(), "test"));
+
+		StopWatch watch = new StopWatch("Lazy-Load Performance");
+
+		int sequenceSize = 1000;
+
+		performLazyLoadEagerTest(watch, sequenceSize, true);
+
+		performLazyLoadEagerTest(watch, sequenceSize, false);
+
+		System. out .println(watch.prettyPrint()); // checkstyle
+	}
+
+	private void performLazyLoadEagerTest(StopWatch watch, int sequenceSize, boolean lazyLoad) {
+		ClassPathXmlApplicationContext context =
+				new ClassPathXmlApplicationContext("mongo-aggregator-configurable-config.xml", getClass());
+		context.refresh();
+
+		AbstractMessageGroupStore messageGroupStore = context.getBean("mongoStore", AbstractMessageGroupStore.class);
+		messageGroupStore.setLazyLoadMessageGroups(lazyLoad);
+		MessageChannel input = context.getBean("inputChannel", MessageChannel.class);
+		QueueChannel output = context.getBean("outputChannel", QueueChannel.class);
+
+		watch.start(lazyLoad ? "Lazy-Load" : "Eager");
+
+		for (int i = 0; i < sequenceSize; i++) {
+			input.send(MessageBuilder.withPayload("" + i)
+					.setCorrelationId(1)
+					.build());
+		}
+
+		assertNotNull(output.receive(20000));
+
+		watch.stop();
+
+		context.close();
 	}
 
 	@Test

spring-integration-mongodb/src/test/java/org/springframework/integration/mongodb/store/MongoDbMessageGroupStoreTests.java

@@ -16,13 +16,14 @@
 
 package org.springframework.integration.mongodb.store;
 
-import com.mongodb.MongoClient;
 import org.junit.Test;
 
 import org.springframework.data.mongodb.core.SimpleMongoDbFactory;
 import org.springframework.integration.mongodb.rules.MongoDbAvailable;
 import org.springframework.integration.store.MessageStore;
 
+import com.mongodb.MongoClient;
+
 /**
  * @author Oleg Zhurakousky
  * @author Gary Russell
@@ -33,7 +34,8 @@ public class MongoDbMessageGroupStoreTests extends AbstractMongoDbMessageGroupSt
 
 	@Override
 	protected MongoDbMessageStore getMessageGroupStore() throws Exception {
-		MongoDbMessageStore mongoDbMessageStore = new MongoDbMessageStore(new SimpleMongoDbFactory(new MongoClient(), "test"));
+		MongoDbMessageStore mongoDbMessageStore =
+				new MongoDbMessageStore(new SimpleMongoDbFactory(new MongoClient(), "test"));
 		mongoDbMessageStore.afterPropertiesSet();
 		return mongoDbMessageStore;
 	}

spring-integration-mongodb/src/test/java/org/springframework/integration/mongodb/store/mongo-aggregator-config.xml

@@ -5,7 +5,8 @@
 	xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
 		http://www.springframework.org/schema/integration http://www.springframework.org/schema/integration/spring-integration.xsd">
 
-	<int:aggregator input-channel="inputChannel" output-channel="outputChannel" message-store="mongoStore"/>
+	<int:aggregator input-channel="inputChannel" output-channel="outputChannel" message-store="mongoStore"
+					release-strategy-expression="size() == 10"/>
 
 	<int:channel id="outputChannel">
 		<int:queue/>

spring-integration-mongodb/src/test/java/org/springframework/integration/mongodb/store/mongo-aggregator-configurable-config.xml

@@ -5,7 +5,8 @@
 	xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
 		http://www.springframework.org/schema/integration http://www.springframework.org/schema/integration/spring-integration.xsd">
 
-	<int:aggregator input-channel="inputChannel" output-channel="outputChannel" message-store="mongoStore"/>
+	<int:aggregator input-channel="inputChannel" output-channel="outputChannel" message-store="mongoStore"
+					release-strategy-expression="size() == 10"/>
 
 	<int:channel id="outputChannel">
 		<int:queue/>

spring-integration-mongodb/src/test/resources/log4j.properties

@@ -0,0 +1,8 @@
+log4j.rootCategory=WARN, stdout
+
+log4j.appender.stdout=org.apache.log4j.ConsoleAppender
+log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
+log4j.appender.stdout.layout.ConversionPattern=%d{HH:mm:ss.SSS} %-5p [%t][%c] %m%n
+
+log4j.category.org.springframework.integration=WARN
+log4j.category.org.springframework.integration.mongodb=INFO