Spring/spring-integration
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

INT-4116: Introduce FileAggregator (#3511)

Browse Source 
* INT-4116: Introduce FileAggregator

JIRA: https://jira.spring.io/browse/INT-4116

* Implement a `FileSplitter.FileMarker`-based aggregation strategies
and utilize them in a general `FileAggregator` component
* Make `HeaderAttributeCorrelationStrategy.attributeName` as `final`; add `Assert.notEmpty()`
* Fix `AggregatorFactoryBean` and `AggregatorSpec` to parse the provided processor
for possible `CorrelationStrategy` and/or `ReleaseStrategy`
* Introduce short-cut methods into Java & Kotlin DSL for an `aggregate()` configuration
* Introduce a `FileHeaders.LINE_COUNT` for header to be populated in the `FileSplitter`.
We need this info in the `FileAggregator` to avoid possible overhead with JSON deserialization
of the `FileSplitter.FileMarker` messages
* Test and document the feature
* Improve `FileSplitter` doc for code block switch (tabs)

* * Rework FileAggregator do not use Java Streams
This commit is contained in:
Artem Bilan
2021-03-16 15:06:36 -04:00
committed by GitHub
parent be2a698618
commit 4342586361
18 changed files with 719 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
spring-integration-file/src/main/java/org/springframework/integration/file/FileHeaders.java
View File

@@ -47,6 +47,11 @@ public abstract class FileHeaders {
*/
public static final String MARKER = PREFIX + "marker";
/**
* The line count for END marker message after splitting
*/
public static final String LINE_COUNT = PREFIX + "lineCount";
/**
* A remote file information representation
*/

60
spring-integration-file/src/main/java/org/springframework/integration/file/aggregator/FileAggregatingMessageGroupProcessor.java Normal file
View File

@@ -0,0 +1,60 @@
/*
* Copyright 2021 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.file.aggregator;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
import org.springframework.integration.aggregator.AbstractAggregatingMessageGroupProcessor;
import org.springframework.integration.file.FileHeaders;
import org.springframework.integration.store.MessageGroup;
import org.springframework.messaging.Message;
/**
* An {@link AbstractAggregatingMessageGroupProcessor} implementation for file content collecting
* previously splitted by the {@link org.springframework.integration.file.splitter.FileSplitter}
* with the {@code markers} option turned on.
* <p>
* If no file markers present in the {@link MessageGroup}, then behavior of this processor is
* similar to the {@link org.springframework.integration.aggregator.DefaultAggregatingMessageGroupProcessor}.
* <p>
* When no file content (only file markers are grouped), this processor emits an empty {@link ArrayList}.
* Note: with no file content and markers turned off,
* the {@link org.springframework.integration.file.splitter.FileSplitter} doesn't emit any messages
* for possible aggregation downstream.
*
* @author Artem Bilan
*
* @since 5.5
*/
public class FileAggregatingMessageGroupProcessor extends AbstractAggregatingMessageGroupProcessor {
@Override
protected Object aggregatePayloads(MessageGroup group, Map<String, Object> defaultHeaders) {
Collection<Message<?>> messages = group.getMessages();
List<Object> payloads = new ArrayList<>(messages.size() - 2);
for (Message<?> message : messages) {
if (!message.getHeaders().containsKey(FileHeaders.MARKER)) {
payloads.add(message.getPayload());
}
}
return payloads;
}
}

78
spring-integration-file/src/main/java/org/springframework/integration/file/aggregator/FileAggregator.java Normal file
View File

@@ -0,0 +1,78 @@
/*
* Copyright 2021 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.file.aggregator;
import org.springframework.beans.BeansException;
import org.springframework.beans.factory.BeanFactory;
import org.springframework.beans.factory.BeanFactoryAware;
import org.springframework.integration.aggregator.CorrelationStrategy;
import org.springframework.integration.aggregator.HeaderAttributeCorrelationStrategy;
import org.springframework.integration.aggregator.MessageGroupProcessor;
import org.springframework.integration.aggregator.ReleaseStrategy;
import org.springframework.integration.file.FileHeaders;
import org.springframework.integration.file.splitter.FileSplitter;
import org.springframework.integration.store.MessageGroup;
import org.springframework.messaging.Message;
/**
* A convenient component to utilize a {@link FileSplitter.FileMarker}-based aggregation logic.
* Implements all three {@link CorrelationStrategy}, {@link ReleaseStrategy} and {@link MessageGroupProcessor}
* for runtime optimization.
* Delegates to {@link HeaderAttributeCorrelationStrategy} with {@link FileHeaders#FILENAME} attribute,
* {@link FileMarkerReleaseStrategy} and {@link FileAggregatingMessageGroupProcessor}, respectively.
* <p>
* The default {@link FileSplitter} behavior with markers enabled is do not provide a sequence details
* headers, therefore correlation in this aggregator implementation is done by the {@link FileHeaders#FILENAME}
* header which is still populated by the {@link FileSplitter} for each line emitted, including
* {@link FileSplitter.FileMarker} messages.
* <p>
* If default behavior of this component does not satisfy the target logic, it is recommended to
* configure an aggregator with individual strategies.
*
* @author Artem Bilan
*
* @since 5.5
*/
public class FileAggregator implements CorrelationStrategy, ReleaseStrategy, MessageGroupProcessor, BeanFactoryAware {
private final CorrelationStrategy correlationStrategy = new HeaderAttributeCorrelationStrategy(FileHeaders.FILENAME);
private final FileMarkerReleaseStrategy releaseStrategy = new FileMarkerReleaseStrategy();
private final FileAggregatingMessageGroupProcessor groupProcessor = new FileAggregatingMessageGroupProcessor();
@Override
public void setBeanFactory(BeanFactory beanFactory) throws BeansException {
this.groupProcessor.setBeanFactory(beanFactory);
}
@Override
public Object getCorrelationKey(Message<?> message) {
return this.correlationStrategy.getCorrelationKey(message);
}
@Override
public boolean canRelease(MessageGroup group) {
return this.releaseStrategy.canRelease(group);
}
@Override
public Object processMessageGroup(MessageGroup group) {
return this.groupProcessor.processMessageGroup(group);
}
}

63
spring-integration-file/src/main/java/org/springframework/integration/file/aggregator/FileMarkerReleaseStrategy.java Normal file
View File

@@ -0,0 +1,63 @@
/*
* Copyright 2021 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.file.aggregator;
import java.util.Collection;
import org.springframework.integration.aggregator.ReleaseStrategy;
import org.springframework.integration.file.FileHeaders;
import org.springframework.integration.file.splitter.FileSplitter;
import org.springframework.integration.store.MessageGroup;
import org.springframework.messaging.Message;
import org.springframework.messaging.MessageHeaders;
/**
* A {@link ReleaseStrategy} which makes a decision based on the presence of
* {@link org.springframework.integration.file.splitter.FileSplitter.FileMarker.Mark#END}
* message in the group and its {@link org.springframework.integration.file.FileHeaders#LINE_COUNT} header.
*
* @author Artem Bilan
*
* @since 5.5
*/
public class FileMarkerReleaseStrategy implements ReleaseStrategy {
@Override
public boolean canRelease(MessageGroup group) {
int size = group.size();
if (size > 1) { // Need more than only a START marker
Collection<Message<?>> messages = group.getMessages();
for (Message<?> message : messages) {
if (checkForEndMarker(size, message.getHeaders())) {
return true;
}
}
}
return false;
}
private boolean checkForEndMarker(int groupSize, MessageHeaders headers) {
if (FileSplitter.FileMarker.Mark.END.name().equals(headers.get(FileHeaders.MARKER))) {
Long lineCount = headers.get(FileHeaders.LINE_COUNT, Long.class);
return lineCount != null && lineCount == groupSize - 2;
}
else {
return false;
}
}
}

4
spring-integration-file/src/main/java/org/springframework/integration/file/aggregator/package-info.java Normal file
View File

@@ -0,0 +1,4 @@
/**
* Provides support classes for file-based aggregation logic.
*/
package org.springframework.integration.file.aggregator;

10
spring-integration-file/src/main/java/org/springframework/integration/file/splitter/FileSplitter.java
View File

@@ -412,8 +412,14 @@ public class FileSplitter extends AbstractMessageSplitter {
else {
payload = fileMarker;
}
return getMessageBuilderFactory().withPayload(payload)
.setHeader(FileHeaders.MARKER, fileMarker.mark.name());
AbstractIntegrationMessageBuilder<Object> messageBuilder =
getMessageBuilderFactory()
.withPayload(payload)
.setHeader(FileHeaders.MARKER, fileMarker.mark.name());
if (Mark.END.equals(fileMarker.mark)) {
messageBuilder.setHeader(FileHeaders.LINE_COUNT, fileMarker.lineCount);
}
return messageBuilder;
}
@Override

146
spring-integration-file/src/test/java/org/springframework/integration/file/aggregator/FileAggregatorTests.java Normal file
View File

@@ -0,0 +1,146 @@
/*
* Copyright 2021 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.file.aggregator;
import static org.assertj.core.api.Assertions.assertThat;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
import java.util.List;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.ImportResource;
import org.springframework.core.task.TaskExecutor;
import org.springframework.integration.IntegrationMessageHeaderAccessor;
import org.springframework.integration.config.EnableIntegration;
import org.springframework.integration.dsl.IntegrationFlow;
import org.springframework.integration.file.FileHeaders;
import org.springframework.integration.file.dsl.Files;
import org.springframework.integration.file.splitter.FileSplitter;
import org.springframework.messaging.Message;
import org.springframework.messaging.MessageChannel;
import org.springframework.messaging.PollableChannel;
import org.springframework.messaging.support.GenericMessage;
import org.springframework.test.annotation.DirtiesContext;
import org.springframework.test.context.junit.jupiter.SpringJUnitConfig;
import org.springframework.util.FileCopyUtils;
/**
* @author Artem Bilan
*
* @since 5.5
*/
@SpringJUnitConfig
@DirtiesContext
public class FileAggregatorTests {
@TempDir
static File tmpDir;
static File file;
@Autowired
@Qualifier("fileSplitterAggregatorFlow.input")
MessageChannel fileSplitterAggregatorFlow;
@Autowired
PollableChannel resultChannel;
@Autowired
MessageChannel input;
@Autowired
PollableChannel output;
@BeforeAll
static void setup() throws IOException {
file = new File(tmpDir, "foo.txt");
String content =
"file header\n" +
"first line\n" +
"second line\n" +
"last line";
FileCopyUtils.copy(content.getBytes(StandardCharsets.UTF_8), new FileOutputStream(file, false));
}
@Test
void testFileAggregator() {
this.fileSplitterAggregatorFlow.send(new GenericMessage<>(file));
Message<?> receive = this.resultChannel.receive(10_000);
assertThat(receive).isNotNull();
assertThat(receive.getHeaders())
.containsEntry(FileHeaders.FILENAME, "foo.txt")
.containsEntry(FileHeaders.LINE_COUNT, 3L)
.containsEntry("firstLine", "file header")
.doesNotContainKey(IntegrationMessageHeaderAccessor.CORRELATION_ID);
assertThat(receive.getPayload())
.isInstanceOf(List.class)
.asList()
.contains("SECOND LINE", "LAST LINE", "FIRST LINE");
}
@Test
void testFileAggregatorXmlConfig() {
this.input.send(new GenericMessage<>(file));
Message<?> receive = this.output.receive(10_000);
assertThat(receive).isNotNull();
assertThat(receive.getHeaders())
.containsEntry(FileHeaders.FILENAME, "foo.txt")
.containsEntry(FileHeaders.LINE_COUNT, 4L)
.doesNotContainKeys("firstLine", IntegrationMessageHeaderAccessor.CORRELATION_ID);
assertThat(receive.getPayload())
.isInstanceOf(List.class)
.asList()
.containsExactly("file header", "first line", "second line", "last line");
}
@Configuration
@EnableIntegration
@ImportResource("org/springframework/integration/file/aggregator/FileAggregatorTests.xml")
public static class Config {
@Bean
public IntegrationFlow fileSplitterAggregatorFlow(TaskExecutor taskExecutor) {
return f -> f
.split(Files.splitter()
.markers()
.firstLineAsHeader("firstLine"))
.channel(c -> c.executor(taskExecutor))
.filter(payload -> !(payload instanceof FileSplitter.FileMarker),
e -> e.discardChannel("aggregatorChannel"))
.<String, String>transform(String::toUpperCase)
.channel("aggregatorChannel")
.aggregate(new FileAggregator())
.channel(c -> c.queue("resultChannel"));
}
}
}

21
spring-integration-file/src/test/java/org/springframework/integration/file/aggregator/FileAggregatorTests.xml Normal file
View File

@@ -0,0 +1,21 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:int-file="http://www.springframework.org/schema/integration/file"
xmlns:int="http://www.springframework.org/schema/integration"
xsi:schemaLocation="http://www.springframework.org/schema/beans https://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/integration https://www.springframework.org/schema/integration/spring-integration.xsd
http://www.springframework.org/schema/integration/file https://www.springframework.org/schema/integration/file/spring-integration-file.xsd">
<int:chain input-channel="input" output-channel="output">
<int-file:splitter markers="true"/>
<int:aggregator>
<bean class="org.springframework.integration.file.aggregator.FileAggregator"/>
</int:aggregator>
</int:chain>
<int:channel id="output">
<int:queue/>
</int:channel>
</beans>

117
spring-integration-file/src/test/kotlin/org/springframework/integration/kotlin/file/aggregator/KotlinFileAggregatorTests.kt Normal file
View File

@@ -0,0 +1,117 @@
/*
* Copyright 2021 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.kotlin.file.aggregator
import assertk.all
import assertk.assertThat
import assertk.assertions.*
import org.junit.jupiter.api.BeforeAll
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.io.TempDir
import org.springframework.beans.factory.annotation.Autowired
import org.springframework.beans.factory.annotation.Qualifier
import org.springframework.context.annotation.Bean
import org.springframework.context.annotation.Configuration
import org.springframework.core.task.TaskExecutor
import org.springframework.integration.IntegrationMessageHeaderAccessor
import org.springframework.integration.config.EnableIntegration
import org.springframework.integration.dsl.integrationFlow
import org.springframework.integration.file.FileHeaders
import org.springframework.integration.file.aggregator.FileAggregator
import org.springframework.integration.file.dsl.Files
import org.springframework.integration.file.splitter.FileSplitter.FileMarker
import org.springframework.messaging.MessageChannel
import org.springframework.messaging.PollableChannel
import org.springframework.messaging.support.GenericMessage
import org.springframework.test.annotation.DirtiesContext
import org.springframework.test.context.junit.jupiter.SpringJUnitConfig
import org.springframework.util.FileCopyUtils
import java.io.File
import java.io.FileOutputStream
/**
* @author Artem Bilan
*
* @since 5.5
*/
@SpringJUnitConfig
@DirtiesContext
class KotlinFileAggregatorTests {
companion object {
lateinit var file: File
@BeforeAll
@JvmStatic
fun setup(@TempDir tmpDir: File) {
file = File(tmpDir, "foo.txt")
val content = """
file header
first line
second line
last line
""".trimIndent()
FileCopyUtils.copy(content.toByteArray(), FileOutputStream(file, false))
}
}
@Autowired
@Qualifier("fileSplitterAggregatorFlow.input")
private lateinit var fileSplitterAggregatorFlow: MessageChannel
@Autowired
private lateinit var resultChannel: PollableChannel
@Test
fun testFileAggregator() {
this.fileSplitterAggregatorFlow.send(GenericMessage(file))
val receive = this.resultChannel.receive(10000)
assertThat(receive).isNotNull()
assertThat(receive.headers)
.all {
contains(FileHeaders.FILENAME, "foo.txt")
contains(FileHeaders.LINE_COUNT, 3L)
contains("firstLine", "file header")
doesNotContain(IntegrationMessageHeaderAccessor.CORRELATION_ID, null)
}
assertThat(receive.payload)
.isInstanceOf(MutableList::class.java)
.containsAll("SECOND LINE", "LAST LINE", "FIRST LINE")
}
@Configuration
@EnableIntegration
class Config {
@Bean
fun fileSplitterAggregatorFlow(taskExecutor: TaskExecutor?) =
integrationFlow {
split(Files.splitter().markers().firstLineAsHeader("firstLine"))
channel { executor(taskExecutor) }
filter<Any>({ it !is FileMarker }) { discardChannel("aggregatorChannel") }
transform(String::toUpperCase)
channel("aggregatorChannel")
aggregate(FileAggregator())
channel { queue("resultChannel") }
}
}
}