Introduce a MessageGroup.condition (#3517)

* Introduce a `groupConditionSupplier` for MGS

* Add a `MessageGroup.condition` option
* Add a `MessageGroupStore.conditionSupplier` option
* Use it from the `SimpleMessageStore.addMessagesToGroup()` API
to populate `condition` (if any) into a `MessageGroup`
* Introduce a `GroupConditionProvider` contract to be implemented
on those `ReleaseStrategy` contracts which could be aware of group condition
* Populated a `GroupConditionProvider.getGroupConditionSupplier()`
into a `MessageGroupStore` from the `AbstractCorrelatingMessageHandler`
for end-user convenience
* Rework a `FileMarkerReleaseStrategy` to implement a `GroupConditionProvider`
to provide a function which produces a condition from `file_lineCount` header
of the `END` marker message
* Make the `FileMarkerReleaseStrategy` logic already based on the condition from a group
* Delegate `GroupConditionProvider` from the `FileAggregator`
* Add test for empty file aggregation

* * Implement `condition` in the `AbstractKeyValueMessageStore` and `MongoDbMessageStore`
* Test `condition` for `mongo-aggregator-config.xml` and `FileAggregatorTests` against GemFire

* * Implement `condition` in the `ConfigurableMongoDbMessageStore`

* * Implement `condition` in the `JdbcMessageStore`
* `FileAggregatorTests` against `JdbcMessageStore`
* Refactor `JdbcMessageStore` for better handling of message group metadata
* Remove unused `MARKED` column in the DDL in favor of newly introduced `CONDITION`

* * Add docs for message group condition

* * Move `conditionSupplier` option from MGS to AbstractCorrelatingMH
* Make it as a `BiFunction` to propagate existing condition alongside with the
message to consult
* Expose `groupConditionSupplier` in Java & XML DSLs

* * Fix language in docs
This commit is contained in:
Artem Bilan
2021-03-23 14:23:46 -04:00
committed by GitHub
parent 7ee3db9ca5
commit 7e9552974c
39 changed files with 533 additions and 151 deletions

View File

@@ -27,6 +27,7 @@ import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ScheduledFuture;
import java.util.concurrent.locks.Lock;
import java.util.function.BiFunction;
import org.aopalliance.aop.Advice;
@@ -158,6 +159,8 @@ public abstract class AbstractCorrelatingMessageHandler extends AbstractMessageP
private volatile boolean running;
private BiFunction<Message<?>, String, String> groupConditionSupplier;
public AbstractCorrelatingMessageHandler(MessageGroupProcessor processor, MessageGroupStore store,
CorrelationStrategy correlationStrategy, ReleaseStrategy releaseStrategy) {
@@ -361,6 +364,17 @@ public abstract class AbstractCorrelatingMessageHandler extends AbstractMessageP
this.expireDuration = expireDuration;
}
/**
* Configure a {@link BiFunction} to supply a group condition from a message to be added to the group.
* The {@code null} result from the function will reset a condition set before.
* @param conditionSupplier the function to supply a group condition from a message to be added to the group.
* @since 5.5
* @see GroupConditionProvider
*/
public void setGroupConditionSupplier(BiFunction<Message<?>, String, String> conditionSupplier) {
this.groupConditionSupplier = conditionSupplier;
}
@Override
public void setApplicationEventPublisher(ApplicationEventPublisher applicationEventPublisher) {
this.applicationEventPublisher = applicationEventPublisher;
@@ -407,6 +421,10 @@ public abstract class AbstractCorrelatingMessageHandler extends AbstractMessageP
*/
this.lockRegistrySet = true;
this.forceReleaseProcessor = createGroupTimeoutProcessor();
if (this.releaseStrategy instanceof GroupConditionProvider) {
this.groupConditionSupplier = ((GroupConditionProvider) this.releaseStrategy).getGroupConditionSupplier();
}
}
private MessageGroupProcessor createGroupTimeoutProcessor() {
@@ -441,6 +459,11 @@ public abstract class AbstractCorrelatingMessageHandler extends AbstractMessageP
return this.releaseStrategy;
}
@Nullable
protected BiFunction<Message<?>, String, String> getGroupConditionSupplier() {
return this.groupConditionSupplier;
}
@Override
public MessageChannel getDiscardChannel() {
String channelName = this.discardChannelName;
@@ -542,6 +565,8 @@ public abstract class AbstractCorrelatingMessageHandler extends AbstractMessageP
this.logger.trace(() -> "Adding message to group [ " + messageGroupToLog + "]");
messageGroup = store(correlationKey, message);
setGroupConditionIfAny(message, messageGroup);
if (this.releaseStrategy.canRelease(messageGroup)) {
Collection<Message<?>> completedMessages = null;
try {
@@ -579,6 +604,14 @@ public abstract class AbstractCorrelatingMessageHandler extends AbstractMessageP
}
}
private void setGroupConditionIfAny(Message<?> message, MessageGroup messageGroup) {
if (this.groupConditionSupplier != null) {
String condition = this.groupConditionSupplier.apply(message, messageGroup.getCondition());
this.messageStore.setGroupCondition(messageGroup.getGroupId(), condition);
messageGroup.setCondition(condition);
}
}
protected boolean isExpireGroupsUponCompletion() {
return false;
}

View File

@@ -0,0 +1,39 @@
/*
* Copyright 2021 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.aggregator;
import java.util.function.BiFunction;
import org.springframework.messaging.Message;
/**
* A contract which can be implemented on the {@link ReleaseStrategy}
* and used in the {@link AbstractCorrelatingMessageHandler} to
* populate the provided group condition supplier.
*
* @author Artem Bilan
*
* @since 5.5
*
* @see AbstractCorrelatingMessageHandler#setGroupConditionSupplier(BiFunction)
*/
@FunctionalInterface
public interface GroupConditionProvider {
BiFunction<Message<?>, String, String> getGroupConditionSupplier();
}

View File

@@ -19,6 +19,7 @@ package org.springframework.integration.config;
import java.time.Duration;
import java.util.List;
import java.util.Map;
import java.util.function.BiFunction;
import java.util.function.Function;
import org.aopalliance.aop.Advice;
@@ -36,6 +37,7 @@ import org.springframework.integration.store.MessageGroup;
import org.springframework.integration.store.MessageGroupStore;
import org.springframework.integration.support.locks.LockRegistry;
import org.springframework.integration.util.JavaUtils;
import org.springframework.messaging.Message;
import org.springframework.messaging.MessageChannel;
import org.springframework.messaging.MessageHandler;
import org.springframework.scheduling.TaskScheduler;
@@ -98,6 +100,8 @@ public class AggregatorFactoryBean extends AbstractSimpleMessageHandlerFactoryBe
private Function<MessageGroup, Map<String, Object>> headersFunction;
private BiFunction<Message<?>, String, String> groupConditionSupplier;
public void setProcessorBean(Object processorBean) {
this.processorBean = processorBean;
}
@@ -187,6 +191,10 @@ public class AggregatorFactoryBean extends AbstractSimpleMessageHandlerFactoryBe
this.expireDuration = expireDuration;
}
public void setGroupConditionSupplier(BiFunction<Message<?>, String, String> groupConditionSupplier) {
this.groupConditionSupplier = groupConditionSupplier;
}
@Override
protected AggregatingMessageHandler createHandler() {
MessageGroupProcessor outputProcessor;
@@ -233,6 +241,7 @@ public class AggregatorFactoryBean extends AbstractSimpleMessageHandlerFactoryBe
.acceptIfNotNull(this.releaseLockBeforeSend, aggregator::setReleaseLockBeforeSend)
.acceptIfNotNull(this.expireDuration,
(duration) -> aggregator.setExpireDuration(Duration.ofMillis(duration)))
.acceptIfNotNull(this.groupConditionSupplier, aggregator::setGroupConditionSupplier)
.acceptIfNotNull(this.expireTimeout, aggregator::setExpireTimeout);
return aggregator;

View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2020 the original author or authors.
* Copyright 2002-2021 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -104,6 +104,8 @@ public abstract class AbstractCorrelatingMessageHandlerParser extends AbstractCo
IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element, "expire-timeout");
IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element, "expire-duration",
"expireDurationMillis");
IntegrationNamespaceUtils.setReferenceIfAttributeDefined(builder, element, "group-condition-supplier");
}
}

View File

@@ -20,6 +20,7 @@ import java.time.Duration;
import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
import java.util.function.BiFunction;
import java.util.function.Function;
import org.aopalliance.aop.Advice;
@@ -36,6 +37,7 @@ import org.springframework.integration.expression.ValueExpression;
import org.springframework.integration.store.MessageGroup;
import org.springframework.integration.store.MessageGroupStore;
import org.springframework.integration.support.locks.LockRegistry;
import org.springframework.messaging.Message;
import org.springframework.messaging.MessageChannel;
import org.springframework.scheduling.TaskScheduler;
import org.springframework.util.Assert;
@@ -397,4 +399,15 @@ public abstract class CorrelationHandlerSpec<S extends CorrelationHandlerSpec<S,
return _this();
}
/**
* Configure a {@link BiFunction} to supply a group condition from a message to be added to the group.
* The {@code null} result from the function will reset a condition set before.
* @param conditionSupplier the function to supply a group condition from a message to be added to the group.
* @since 5.5
*/
public S groupConditionSupplier(BiFunction<Message<?>, String, String> conditionSupplier) {
this.handler.setGroupConditionSupplier(conditionSupplier);
return _this();
}
}

View File

@@ -180,6 +180,7 @@ public abstract class AbstractKeyValueMessageStore extends AbstractMessageGroupS
.create(this, groupId, metadata.getTimestamp(), metadata.isComplete());
messageGroup.setLastModified(metadata.getLastModified());
messageGroup.setLastReleasedMessageSequenceNumber(metadata.getLastReleasedMessageSequenceNumber());
messageGroup.setCondition(metadata.getCondition());
return messageGroup;
}
else {
@@ -294,6 +295,15 @@ public abstract class AbstractKeyValueMessageStore extends AbstractMessageGroupS
}
}
@Override
public void setGroupCondition(Object groupId, String condition) {
MessageGroupMetadata metadata = getGroupMetadata(groupId);
if (metadata != null) {
metadata.setCondition(condition);
doStore(this.groupPrefix + groupId, metadata);
}
}
@Override
public void setLastReleasedSequenceNumberForGroup(Object groupId, int sequenceNumber) {
Assert.notNull(groupId, GROUP_ID_MUST_NOT_BE_NULL);

View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2019 the original author or authors.
* Copyright 2002-2021 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -47,10 +47,10 @@ public abstract class AbstractMessageGroupStore extends AbstractBatchingMessageG
private final MessageGroupFactory persistentMessageGroupFactory =
new SimpleMessageGroupFactory(SimpleMessageGroupFactory.GroupType.PERSISTENT);
private volatile boolean timeoutOnIdle;
private boolean lazyLoadMessageGroups = true;
private boolean timeoutOnIdle;
protected AbstractMessageGroupStore() {
}
@@ -111,7 +111,7 @@ public abstract class AbstractMessageGroupStore extends AbstractBatchingMessageG
this.expiryCallbacks.stream()
.anyMatch(UniqueExpiryCallback.class::isInstance);
if (uniqueExpiryCallbackPresent && this.logger.isErrorEnabled()) {
if (uniqueExpiryCallbackPresent) {
this.logger.error("Only one instance of 'UniqueExpiryCallback' can be registered in the " +
"'MessageGroupStore'. Use a separate 'MessageGroupStore' for each aggregator/resequencer.");
}

View File

@@ -19,6 +19,7 @@ package org.springframework.integration.store;
import java.util.Collection;
import java.util.stream.Stream;
import org.springframework.lang.Nullable;
import org.springframework.messaging.Message;
/**
@@ -120,6 +121,21 @@ public interface MessageGroup {
void setLastModified(long lastModified);
/**
* Add a condition statement to this group which can be consulted later on, e.g. from the release strategy.
* @param condition statement which could be consulted later on, e.g. from the release strategy.
* @since 5.5
*/
void setCondition(String condition);
/**
* Return the condition for this group to consult with, e.g. from the release strategy.
* @return the condition for this group to consult with, e.g. from the release strategy.
* @since 5.5
*/
@Nullable
String getCondition();
void clear();
}

View File

@@ -50,6 +50,8 @@ public class MessageGroupMetadata implements Serializable {
private volatile int lastReleasedMessageSequenceNumber;
private volatile String condition;
private MessageGroupMetadata() {
//For Jackson deserialization
}
@@ -129,4 +131,12 @@ public class MessageGroupMetadata implements Serializable {
this.lastReleasedMessageSequenceNumber = lastReleasedMessageSequenceNumber;
}
public String getCondition() {
return this.condition;
}
public void setCondition(String condition) {
this.condition = condition;
}
}

View File

@@ -96,6 +96,17 @@ public interface MessageGroupStore extends BasicMessageGroupStore {
*/
void setLastReleasedSequenceNumberForGroup(Object groupId, int sequenceNumber);
/**
* Add a condition sentence into the group.
* Can be used later on for making some decisions for group, e.g. release strategy
* for correlation handler can consult this condition instead of iterating all
* the messages in group.
* @param groupId The group identifier.
* @param condition The condition to store into the group.
* @since 5.5
*/
void setGroupCondition(Object groupId, String condition);
/**
* @return The iterator of currently accumulated {@link MessageGroup}s.
*/

View File

@@ -27,6 +27,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.springframework.integration.IntegrationMessageHeaderAccessor;
import org.springframework.lang.Nullable;
import org.springframework.messaging.Message;
/**
@@ -175,6 +176,17 @@ class PersistentMessageGroup implements MessageGroup {
this.original.setLastReleasedMessageSequenceNumber(sequenceNumber);
}
@Override
public void setCondition(String condition) {
this.original.setCondition(condition);
}
@Override
@Nullable
public String getCondition() {
return this.original.getCondition();
}
@Override
public void clear() {
this.original.clear();

View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2019 the original author or authors.
* Copyright 2002-2021 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -24,12 +24,13 @@ import java.util.LinkedHashSet;
import java.util.Set;
import org.springframework.integration.IntegrationMessageHeaderAccessor;
import org.springframework.lang.Nullable;
import org.springframework.messaging.Message;
import org.springframework.util.Assert;
/**
* Represents a mutable group of correlated messages that is bound to a certain {@link MessageStore} and group id.
* The group will grow during its lifetime, when messages are <code>add</code>ed to it.
* The group will grow during its lifetime, when messages are {@link #add}ed to it.
* This MessageGroup is thread safe.
*
* @author Iwein Fuld
@@ -56,8 +57,11 @@ public class SimpleMessageGroup implements MessageGroup {
private volatile boolean complete;
@Nullable
private volatile String condition;
public SimpleMessageGroup(Object groupId) {
this(Collections.<Message<?>>emptyList(), groupId);
this(Collections.emptyList(), groupId);
}
public SimpleMessageGroup(Collection<? extends Message<?>> messages, Object groupId) {
@@ -172,6 +176,17 @@ public class SimpleMessageGroup implements MessageGroup {
return this.messages.size();
}
@Override
public void setCondition(String condition) {
this.condition = condition;
}
@Override
@Nullable
public String getCondition() {
return this.condition;
}
@Override
public Message<?> getOne() {
synchronized (this.messages) {

View File

@@ -1,5 +1,5 @@
/*
* Copyright 2002-2019 the original author or authors.
* Copyright 2002-2021 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
@@ -56,12 +56,11 @@ public class SimpleMessageStore extends AbstractMessageGroupStore
private static final String INTERRUPTED_WHILE_OBTAINING_LOCK = "Interrupted while obtaining lock";
private final ConcurrentMap<UUID, Message<?>> idToMessage = new ConcurrentHashMap<UUID, Message<?>>();
private final ConcurrentMap<UUID, Message<?>> idToMessage = new ConcurrentHashMap<>();
private final ConcurrentMap<Object, MessageGroup> groupIdToMessageGroup =
new ConcurrentHashMap<Object, MessageGroup>();
private final ConcurrentMap<Object, MessageGroup> groupIdToMessageGroup = new ConcurrentHashMap<>();
private final ConcurrentMap<Object, UpperBound> groupToUpperBound = new ConcurrentHashMap<Object, UpperBound>();
private final ConcurrentMap<Object, UpperBound> groupToUpperBound = new ConcurrentHashMap<>();
private final int groupCapacity;
@@ -69,14 +68,14 @@ public class SimpleMessageStore extends AbstractMessageGroupStore
private final UpperBound individualUpperBound;
private volatile LockRegistry lockRegistry;
private final long upperBoundTimeout;
private LockRegistry lockRegistry;
private boolean copyOnGet = false;
private volatile boolean isUsed;
private volatile boolean copyOnGet = false;
private final long upperBoundTimeout;
/**
* Creates a SimpleMessageStore with a maximum size limited by the given capacity, or unlimited size if the given
* capacity is less than 1. The capacities are applied independently to messages stored via
@@ -183,7 +182,7 @@ public class SimpleMessageStore extends AbstractMessageGroupStore
public <T> Message<T> addMessage(Message<T> message) {
this.isUsed = true;
if (!this.individualUpperBound.tryAcquire(this.upperBoundTimeout)) {
throw new MessagingException(this.getClass().getSimpleName()
throw new MessagingException(getClass().getSimpleName()
+ " was out of capacity ("
+ this.individualCapacity
+ "), try constructing it with a larger capacity.");
@@ -254,6 +253,7 @@ public class SimpleMessageStore extends AbstractMessageGroupStore
.create(group.getMessages(), groupId, group.getTimestamp(), group.isComplete());
simpleMessageGroup.setLastModified(group.getLastModified());
simpleMessageGroup.setLastReleasedMessageSequenceNumber(group.getLastReleasedMessageSequenceNumber());
simpleMessageGroup.setCondition(group.getCondition());
return simpleMessageGroup;
}
finally {
@@ -353,8 +353,9 @@ public class SimpleMessageStore extends AbstractMessageGroupStore
lock.lockInterruptibly();
try {
MessageGroup group = this.groupIdToMessageGroup.get(groupId);
Assert.notNull(group, MESSAGE_GROUP_FOR_GROUP_ID + groupId + "' " +
"can not be located while attempting to remove Message(s) from the MessageGroup");
Assert.notNull(group,
() -> MESSAGE_GROUP_FOR_GROUP_ID + groupId + "' " +
"can not be located while attempting to remove Message(s) from the MessageGroup");
UpperBound upperBound = this.groupToUpperBound.get(groupId);
Assert.state(upperBound != null, UPPER_BOUND_MUST_NOT_BE_NULL);
boolean modified = false;
@@ -380,7 +381,15 @@ public class SimpleMessageStore extends AbstractMessageGroupStore
@Override
public Iterator<MessageGroup> iterator() {
return new HashSet<MessageGroup>(this.groupIdToMessageGroup.values()).iterator();
return new HashSet<>(this.groupIdToMessageGroup.values()).iterator();
}
@Override
public void setGroupCondition(Object groupId, String condition) {
MessageGroup group = this.groupIdToMessageGroup.get(groupId);
if (group != null) {
group.setCondition(condition);
}
}
@Override
@@ -390,8 +399,9 @@ public class SimpleMessageStore extends AbstractMessageGroupStore
lock.lockInterruptibly();
try {
MessageGroup group = this.groupIdToMessageGroup.get(groupId);
Assert.notNull(group, MESSAGE_GROUP_FOR_GROUP_ID + groupId + "' " +
"can not be located while attempting to set 'lastReleasedSequenceNumber'");
Assert.notNull(group,
() -> MESSAGE_GROUP_FOR_GROUP_ID + groupId + "' " +
"can not be located while attempting to set 'lastReleasedSequenceNumber'");
group.setLastReleasedMessageSequenceNumber(sequenceNumber);
group.setLastModified(System.currentTimeMillis());
}
@@ -412,8 +422,9 @@ public class SimpleMessageStore extends AbstractMessageGroupStore
lock.lockInterruptibly();
try {
MessageGroup group = this.groupIdToMessageGroup.get(groupId);
Assert.notNull(group, MESSAGE_GROUP_FOR_GROUP_ID + groupId + "' " +
"can not be located while attempting to complete the MessageGroup");
Assert.notNull(group,
() -> MESSAGE_GROUP_FOR_GROUP_ID + groupId + "' " +
"can not be located while attempting to complete the MessageGroup");
group.complete();
group.setLastModified(System.currentTimeMillis());
}
@@ -466,8 +477,9 @@ public class SimpleMessageStore extends AbstractMessageGroupStore
lock.lockInterruptibly();
try {
MessageGroup group = this.groupIdToMessageGroup.get(groupId);
Assert.notNull(group, MESSAGE_GROUP_FOR_GROUP_ID + groupId + "' " +
"can not be located while attempting to complete the MessageGroup");
Assert.notNull(group,
() -> MESSAGE_GROUP_FOR_GROUP_ID + groupId + "' " +
"can not be located while attempting to complete the MessageGroup");
group.clear();
group.setLastModified(System.currentTimeMillis());
UpperBound upperBound = this.groupToUpperBound.get(groupId);

View File

@@ -4071,6 +4071,19 @@
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
<xsd:attribute name="group-condition-supplier" type="xsd:string">
<xsd:annotation>
<xsd:appinfo>
<tool:annotation kind="ref">
<tool:expected-type type="java.util.function.BiFunction"/>
</tool:annotation>
</xsd:appinfo>
<xsd:documentation>
A reference to a 'java.util.function.BiFunction' bean that may provide a group
condition evaluated against the message to be added to the group.
</xsd:documentation>
</xsd:annotation>
</xsd:attribute>
</xsd:extension>
</xsd:complexContent>
</xsd:complexType>