Issue #30 : Add S3 inbound streaming channel adapter.
Fixes GH-30 (https://github.com/spring-projects/spring-integration-aws/issues/30) * Add S3InboundStreamingMessageSource and related S3FileInfo * Extend spring-integration-aws-1.1.xsd with s3-inbound-streaming-channel-adapter tag. * Add S3StreamingInboundChannelAdapterParser and s3-inbound-streaming-channel-adapter handler. * Add S3 streaming spring tests. * Add S3 streaming documentation to README.md. Issue #30: Resolve code style and formatting issues Polishing: * Revert some `build.gradle` changes * Add JavaDoc to the `S3FileInfo.getPermissions()`
This commit is contained in:
Christian Tzolov
committed by
Artem Bilan
Artem Bilan
parent
de4dfed908
commit
da77e4b89e
@@ -0,0 +1,43 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<beans xmlns="http://www.springframework.org/schema/beans"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xmlns:int-aws="http://www.springframework.org/schema/integration/aws"
|
||||
xmlns:int="http://www.springframework.org/schema/integration"
|
||||
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
|
||||
http://www.springframework.org/schema/integration/aws http://www.springframework.org/schema/integration/aws/spring-integration-aws.xsd
|
||||
http://www.springframework.org/schema/integration http://www.springframework.org/schema/integration/spring-integration.xsd">
|
||||
|
||||
<bean id="s3" class="org.mockito.Mockito" factory-method="mock">
|
||||
<constructor-arg value="com.amazonaws.services.s3.AmazonS3"/>
|
||||
</bean>
|
||||
|
||||
<bean id="s3SessionFactory" class="org.springframework.integration.aws.support.S3SessionFactory">
|
||||
<constructor-arg ref="s3"/>
|
||||
</bean>
|
||||
|
||||
<bean id="comparator" class="org.mockito.Mockito" factory-method="mock">
|
||||
<constructor-arg value="java.util.Comparator"/>
|
||||
</bean>
|
||||
|
||||
<int:channel id="s3Channel">
|
||||
<int:queue/>
|
||||
</int:channel>
|
||||
|
||||
<bean id="metadataStore" class="org.springframework.integration.metadata.SimpleMetadataStore"/>
|
||||
|
||||
<bean id="acceptOnceFilter" class="org.springframework.integration.aws.support.filters.S3PersistentAcceptOnceFileListFilter">
|
||||
<constructor-arg index="0" ref="metadataStore"/>
|
||||
<constructor-arg index="1" value="streaming"/>
|
||||
</bean>
|
||||
|
||||
<int-aws:s3-inbound-streaming-channel-adapter id="s3Inbound"
|
||||
session-factory="s3SessionFactory"
|
||||
channel="s3Channel"
|
||||
auto-startup="false"
|
||||
comparator="comparator"
|
||||
filter="acceptOnceFilter"
|
||||
remote-directory-expression="foo/bar">
|
||||
<int:poller fixed-rate="1000"/>
|
||||
</int-aws:s3-inbound-streaming-channel-adapter>
|
||||
|
||||
</beans>
|
||||
@@ -0,0 +1,103 @@
|
||||
/*
|
||||
* Copyright 2016 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.integration.aws.config.xml;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import java.lang.reflect.Method;
|
||||
import java.util.Comparator;
|
||||
import java.util.concurrent.atomic.AtomicReference;
|
||||
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.expression.Expression;
|
||||
import org.springframework.integration.aws.inbound.S3InboundStreamingMessageSource;
|
||||
import org.springframework.integration.aws.support.filters.S3PersistentAcceptOnceFileListFilter;
|
||||
import org.springframework.integration.endpoint.SourcePollingChannelAdapter;
|
||||
import org.springframework.integration.file.remote.session.SessionFactory;
|
||||
import org.springframework.integration.file.remote.synchronizer.AbstractInboundFileSynchronizer;
|
||||
import org.springframework.integration.test.util.TestUtils;
|
||||
import org.springframework.messaging.MessageChannel;
|
||||
import org.springframework.test.context.ContextConfiguration;
|
||||
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
|
||||
import org.springframework.util.ReflectionUtils;
|
||||
|
||||
/**
|
||||
* @author Christian Tzolov
|
||||
*/
|
||||
@RunWith(SpringJUnit4ClassRunner.class)
|
||||
@ContextConfiguration
|
||||
public class S3InboundStreamingChannelAdapterParserTests {
|
||||
|
||||
@Autowired
|
||||
private SourcePollingChannelAdapter s3Inbound;
|
||||
|
||||
@Autowired
|
||||
private Comparator<?> comparator;
|
||||
|
||||
@Autowired
|
||||
private MessageChannel s3Channel;
|
||||
|
||||
@Autowired
|
||||
private S3PersistentAcceptOnceFileListFilter acceptOnceFilter;
|
||||
|
||||
@Autowired
|
||||
private SessionFactory<?> s3SessionFactory;
|
||||
|
||||
@Test
|
||||
public void testS3StreamingInboundChannelAdapterComplete() throws Exception {
|
||||
|
||||
assertThat(TestUtils.getPropertyValue(this.s3Inbound, "autoStartup", Boolean.class)).isFalse();
|
||||
assertThat(this.s3Inbound.getComponentName()).isEqualTo("s3Inbound");
|
||||
assertThat(this.s3Inbound.getComponentType()).isEqualTo("aws:s3-inbound-streaming-channel-adapter");
|
||||
assertThat(TestUtils.getPropertyValue(this.s3Inbound, "outputChannel")).isSameAs(this.s3Channel);
|
||||
|
||||
S3InboundStreamingMessageSource source = TestUtils.getPropertyValue(this.s3Inbound, "source",
|
||||
S3InboundStreamingMessageSource.class);
|
||||
|
||||
assertThat(TestUtils.getPropertyValue(source, "remoteDirectoryExpression", Expression.class)
|
||||
.getExpressionString())
|
||||
.isEqualTo("foo/bar");
|
||||
|
||||
assertThat(TestUtils.getPropertyValue(source, "comparator")).isSameAs(this.comparator);
|
||||
String remoteFileSeparator = (String) TestUtils.getPropertyValue(source, "remoteFileSeparator");
|
||||
assertThat(remoteFileSeparator).isNotNull();
|
||||
assertThat(remoteFileSeparator).isEqualTo("/");
|
||||
|
||||
S3PersistentAcceptOnceFileListFilter filter = TestUtils.getPropertyValue(source, "filter",
|
||||
S3PersistentAcceptOnceFileListFilter.class);
|
||||
assertThat(filter).isSameAs(this.acceptOnceFilter);
|
||||
assertThat(TestUtils.getPropertyValue(source, "remoteFileTemplate.sessionFactory"))
|
||||
.isSameAs(this.s3SessionFactory);
|
||||
|
||||
final AtomicReference<Method> genMethod = new AtomicReference<Method>();
|
||||
ReflectionUtils.doWithMethods(AbstractInboundFileSynchronizer.class, new ReflectionUtils.MethodCallback() {
|
||||
|
||||
@Override
|
||||
public void doWith(Method method) throws IllegalArgumentException, IllegalAccessException {
|
||||
if ("generateLocalFileName".equals(method.getName())) {
|
||||
method.setAccessible(true);
|
||||
genMethod.set(method);
|
||||
}
|
||||
}
|
||||
|
||||
});
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,195 @@
|
||||
/*
|
||||
* Copyright 2016 the original author or authors.
|
||||
*
|
||||
* Licensed under the Apache License, Version 2.0 (the "License");
|
||||
* you may not use this file except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.springframework.integration.aws.inbound;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.BDDMockito.willAnswer;
|
||||
import static org.mockito.Matchers.any;
|
||||
|
||||
import java.io.File;
|
||||
import java.io.FileInputStream;
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Date;
|
||||
import java.util.List;
|
||||
|
||||
import org.apache.commons.io.IOUtils;
|
||||
import org.junit.BeforeClass;
|
||||
import org.junit.ClassRule;
|
||||
import org.junit.Test;
|
||||
import org.junit.rules.TemporaryFolder;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.mockito.Mockito;
|
||||
import org.mockito.invocation.InvocationOnMock;
|
||||
import org.mockito.stubbing.Answer;
|
||||
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.integration.annotation.InboundChannelAdapter;
|
||||
import org.springframework.integration.annotation.Poller;
|
||||
import org.springframework.integration.aws.support.S3RemoteFileTemplate;
|
||||
import org.springframework.integration.aws.support.S3SessionFactory;
|
||||
import org.springframework.integration.aws.support.filters.S3PersistentAcceptOnceFileListFilter;
|
||||
import org.springframework.integration.channel.QueueChannel;
|
||||
import org.springframework.integration.config.EnableIntegration;
|
||||
import org.springframework.integration.file.FileHeaders;
|
||||
import org.springframework.integration.metadata.SimpleMetadataStore;
|
||||
import org.springframework.messaging.Message;
|
||||
import org.springframework.messaging.PollableChannel;
|
||||
import org.springframework.test.annotation.DirtiesContext;
|
||||
import org.springframework.test.context.ContextConfiguration;
|
||||
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
|
||||
import org.springframework.util.FileCopyUtils;
|
||||
|
||||
import com.amazonaws.services.s3.AmazonS3;
|
||||
import com.amazonaws.services.s3.model.ListObjectsRequest;
|
||||
import com.amazonaws.services.s3.model.ObjectListing;
|
||||
import com.amazonaws.services.s3.model.S3Object;
|
||||
import com.amazonaws.services.s3.model.S3ObjectSummary;
|
||||
|
||||
/**
|
||||
* @author Christian Tzolov
|
||||
*/
|
||||
@RunWith(SpringJUnit4ClassRunner.class)
|
||||
@ContextConfiguration
|
||||
@DirtiesContext
|
||||
public class S3InboundStreamingChannelAdapterTests {
|
||||
|
||||
@ClassRule
|
||||
public static final TemporaryFolder TEMPORARY_FOLDER = new TemporaryFolder();
|
||||
|
||||
private static final String S3_BUCKET = "S3_BUCKET";
|
||||
|
||||
private static List<S3Object> S3_OBJECTS;
|
||||
|
||||
@Autowired
|
||||
private PollableChannel s3FilesChannel;
|
||||
|
||||
@BeforeClass
|
||||
public static void setup() throws IOException {
|
||||
File remoteFolder = TEMPORARY_FOLDER.newFolder("remote");
|
||||
|
||||
File aFile = new File(remoteFolder, "a.test");
|
||||
FileCopyUtils.copy("Hello".getBytes(), aFile);
|
||||
File bFile = new File(remoteFolder, "b.test");
|
||||
FileCopyUtils.copy("Bye".getBytes(), bFile);
|
||||
|
||||
S3_OBJECTS = new ArrayList<>();
|
||||
|
||||
for (File file : remoteFolder.listFiles()) {
|
||||
S3Object s3Object = new S3Object();
|
||||
s3Object.setBucketName(S3_BUCKET);
|
||||
s3Object.setKey("subdir/" + file.getName());
|
||||
s3Object.setObjectContent(new FileInputStream(file));
|
||||
|
||||
S3_OBJECTS.add(s3Object);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testS3InboundChannelAdapter() throws IOException {
|
||||
Message<?> message = this.s3FilesChannel.receive(10000);
|
||||
assertThat(message).isNotNull();
|
||||
assertThat(message.getPayload()).isInstanceOf(InputStream.class);
|
||||
assertThat(message.getHeaders().get(FileHeaders.REMOTE_FILE)).isEqualTo("subdir/a.test");
|
||||
|
||||
InputStream inputStreamA = (InputStream) message.getPayload();
|
||||
assertThat(inputStreamA).isNotNull();
|
||||
assertThat(IOUtils.toString(inputStreamA)).isEqualTo("Hello");
|
||||
|
||||
message = this.s3FilesChannel.receive(10000);
|
||||
assertThat(message).isNotNull();
|
||||
assertThat(message.getPayload()).isInstanceOf(InputStream.class);
|
||||
assertThat(message.getHeaders().get(FileHeaders.REMOTE_FILE)).isEqualTo("subdir/b.test");
|
||||
InputStream inputStreamB = (InputStream) message.getPayload();
|
||||
assertThat(IOUtils.toString(inputStreamB)).isEqualTo("Bye");
|
||||
|
||||
assertThat(this.s3FilesChannel.receive(10000)).isNull();
|
||||
}
|
||||
|
||||
@Configuration
|
||||
@EnableIntegration
|
||||
public static class Config {
|
||||
|
||||
@Bean
|
||||
public AmazonS3 amazonS3() {
|
||||
AmazonS3 amazonS3 = Mockito.mock(AmazonS3.class);
|
||||
|
||||
willAnswer(new Answer<ObjectListing>() {
|
||||
|
||||
@Override
|
||||
public ObjectListing answer(InvocationOnMock invocation) throws Throwable {
|
||||
ObjectListing objectListing = new ObjectListing();
|
||||
List<S3ObjectSummary> objectSummaries = objectListing.getObjectSummaries();
|
||||
for (S3Object s3Object : S3_OBJECTS) {
|
||||
S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
|
||||
s3ObjectSummary.setBucketName(S3_BUCKET);
|
||||
s3ObjectSummary.setKey(s3Object.getKey());
|
||||
|
||||
s3ObjectSummary.setLastModified(new Date(new File(s3Object.getKey()).lastModified()));
|
||||
objectSummaries.add(s3ObjectSummary);
|
||||
}
|
||||
return objectListing;
|
||||
}
|
||||
|
||||
}).given(amazonS3).listObjects(any(ListObjectsRequest.class));
|
||||
|
||||
for (final S3Object s3Object : S3_OBJECTS) {
|
||||
willAnswer(new Answer<S3Object>() {
|
||||
|
||||
@Override
|
||||
public S3Object answer(InvocationOnMock invocation) throws Throwable {
|
||||
return s3Object;
|
||||
}
|
||||
|
||||
}).given(amazonS3).getObject(S3_BUCKET, s3Object.getKey());
|
||||
}
|
||||
|
||||
return amazonS3;
|
||||
}
|
||||
|
||||
|
||||
@Bean
|
||||
@InboundChannelAdapter(value = "s3FilesChannel", poller = @Poller(fixedDelay = "100"))
|
||||
public S3InboundStreamingMessageSource s3InboundStreamingMessageSource(AmazonS3 amazonS3) {
|
||||
|
||||
S3SessionFactory s3SessionFactory = new S3SessionFactory(amazonS3);
|
||||
|
||||
S3RemoteFileTemplate s3FileTemplate = new S3RemoteFileTemplate(s3SessionFactory);
|
||||
|
||||
s3FileTemplate.setUseTemporaryFileName(false);
|
||||
|
||||
S3InboundStreamingMessageSource s3MessageSource = new S3InboundStreamingMessageSource(s3FileTemplate);
|
||||
|
||||
s3MessageSource.setRemoteDirectory(S3_BUCKET);
|
||||
|
||||
s3MessageSource.setFilter(new S3PersistentAcceptOnceFileListFilter(new SimpleMetadataStore(),
|
||||
"streaming"));
|
||||
|
||||
return s3MessageSource;
|
||||
}
|
||||
|
||||
@Bean
|
||||
public PollableChannel s3FilesChannel() {
|
||||
return new QueueChannel();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user