Introduce PartitionedChannel (#8617)

* Introduce `PartitionedChannel`

* Implement a `PartitionedChannel` as an extension of the `AbstractExecutorChannel`
* Supply this channel with a `PartitionedDispatcher` which is an extension of the `AbstractDispatcher`
* The target partition is essentially a `UnicastingDispatcher` with a single thead executor

* * Fix language in Javadocs
* Add docs for `PartitionedChannel` into `channel.adoc`
* Pre-populate partitions in the `PartitionedDispatcher`
to ensure a thread number reflection of the partition it is used for (for default `ThreadFactory`)
* Add Javadocs to `public` methods
* Add Java DSL `PartitionedChannelSpec` and respective factory methods into `Channels`
* Use `IntegrationMessageHeaderAccessor.CORRELATION_ID` header as a default partition key

* * Fix language in Javadocs
* Fix Checkstyle violations
* Mark `PartitionedDispatcher.prePopulatedPartitions()` with `synchronized`

* * Populate partitions from `PartitionedDispatcher` ctor
and when a custom `ThreadFactory` is set

* * Clear `executors` in `populatedPartitions()`

* * Bring back `synchronized populatedPartitions()` and use it in the `dispatch()`
This commit is contained in:
Artem Bilan
2023-05-10 15:49:33 -04:00
committed by GitHub
parent 32b6d823c0
commit 5f1e12ea35
8 changed files with 651 additions and 2 deletions

View File

@@ -0,0 +1,161 @@
/*
* Copyright 2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.channel;
import java.util.Collection;
import java.util.HashSet;
import java.util.Set;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.stream.IntStream;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.BeanFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.integration.config.EnableIntegration;
import org.springframework.integration.dsl.IntegrationFlow;
import org.springframework.integration.support.MessageBuilder;
import org.springframework.messaging.Message;
import org.springframework.messaging.MessageChannel;
import org.springframework.messaging.MessageHandler;
import org.springframework.messaging.PollableChannel;
import org.springframework.messaging.support.ExecutorChannelInterceptor;
import org.springframework.messaging.support.GenericMessage;
import org.springframework.test.context.junit.jupiter.SpringJUnitConfig;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.Mockito.mock;
/**
* @author Artem Bilan
*
* @since 6.1
*/
@SpringJUnitConfig
public class PartitionedChannelTests {
@Test
void messagesAreProperlyPartitioned() throws InterruptedException {
PartitionedChannel partitionedChannel =
new PartitionedChannel(2, (message) -> message.getHeaders().get("partitionKey"));
partitionedChannel.setBeanFactory(mock(BeanFactory.class));
partitionedChannel.setBeanName("testPartitionedChannel");
CountDownLatch handleLatch = new CountDownLatch(4);
partitionedChannel.addInterceptor(new ExecutorChannelInterceptor() {
@Override
public void afterMessageHandled(Message<?> message, MessageChannel ch, MessageHandler h, Exception ex) {
handleLatch.countDown();
}
});
partitionedChannel.afterPropertiesSet();
MultiValueMap<String, Message<?>> partitionedMessages = new LinkedMultiValueMap<>();
partitionedChannel.subscribe((message) -> partitionedMessages.add(Thread.currentThread().getName(), message));
partitionedChannel.send(MessageBuilder.withPayload("test1").setHeader("partitionKey", "1").build());
partitionedChannel.send(MessageBuilder.withPayload("test2").setHeader("partitionKey", "2").build());
partitionedChannel.send(MessageBuilder.withPayload("test3").setHeader("partitionKey", "2").build());
partitionedChannel.send(MessageBuilder.withPayload("test4").setHeader("partitionKey", "1").build());
assertThat(handleLatch.await(10, TimeUnit.SECONDS)).isTrue();
assertThat(partitionedMessages).hasSize(2);
partitionedMessages.values()
.forEach(messagesInPartition -> {
assertThat(messagesInPartition).hasSize(2);
assertThat(messagesInPartition.get(0).getHeaders().get("partitionKey"))
.isEqualTo(messagesInPartition.get(1).getHeaders().get("partitionKey"));
});
HashSet<String> allocatedPartitions = new HashSet<>(partitionedMessages.keySet());
partitionedMessages.clear();
CountDownLatch anotherHandleLatch = new CountDownLatch(1);
partitionedChannel.addInterceptor(new ExecutorChannelInterceptor() {
@Override
public void afterMessageHandled(Message<?> message, MessageChannel ch, MessageHandler h, Exception ex) {
anotherHandleLatch.countDown();
}
});
partitionedChannel.send(MessageBuilder.withPayload("test4").setHeader("partitionKey", "3").build());
assertThat(anotherHandleLatch.await(10, TimeUnit.SECONDS)).isTrue();
assertThat(partitionedMessages).hasSize(1);
String partitionForLastMessage = partitionedMessages.keySet().iterator().next();
assertThat(partitionForLastMessage).isIn(allocatedPartitions);
partitionedChannel.destroy();
}
@Autowired
@Qualifier("someFlow.input")
MessageChannel inputChannel;
@Autowired
PollableChannel resultChannel;
@Test
void messagesArePartitionedByCorrelationId() {
this.inputChannel.send(new GenericMessage<>(IntStream.range(0, 5).toArray()));
Message<?> receive = this.resultChannel.receive(10_000);
assertThat(receive).isNotNull()
.extracting(Message::getPayload)
.asList()
.hasSize(5);
@SuppressWarnings("unchecked")
Set<String> strings = new HashSet<>((Collection<? extends String>) receive.getPayload());
assertThat(strings).hasSize(1)
.allMatch(value -> value.startsWith("testChannel-partition-thread-"));
}
@Configuration
@EnableIntegration
public static class TestConfiguration {
@Bean
IntegrationFlow someFlow() {
return f -> f
.split()
.channel(c -> c.partitioned("testChannel", 10))
.transform(p -> Thread.currentThread().getName())
.aggregate()
.channel(c -> c.queue("resultChannel"));
}
}
}