Spring/spring-integration-aws
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add S3 MessageSource implementation

Browse Source 
JIRA: https://jira.spring.io/browse/INTEXT-51,
https://jira.spring.io/browse/INTEXT-194,
https://jira.spring.io/browse/INTEXT-199

* Add S3 Inbound Channel Adapter fully based on the `AbstractInboundFileSynchronizer`,
`AbstractInboundFileSynchronizingMessageSource` implementation.
* Provide the standard SI `RemoteFileTemplate` and `SessionFactory` abstractions implementations.
* Upgrade to Gradle-2.12
* Provide Namespace support
* Ensure everything with tests
* Remove legacy, redundant implementation, infrastructure code for it and test-cases.
Everything in favor of the new implementation.
This commit is contained in:
Artem Bilan
2016-04-04 19:59:02 -04:00
parent 5167b36b4d
commit 29e255cc64
86 changed files with 1283 additions and 7779 deletions
Download Patch File Download Diff File Expand all files Collapse all files

27
src/test/resources/s3-both-customfilegenerator-and-expression.xml
View File

@@ -1,27 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:integration="http://www.springframework.org/schema/integration"
xmlns:int-aws="http://www.springframework.org/schema/integration/aws"
xsi:schemaLocation=
"http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/integration/aws http://www.springframework.org/schema/integration/aws/spring-integration-aws.xsd
http://www.springframework.org/schema/integration http://www.springframework.org/schema/integration/spring-integration.xsd">
<integration:channel id="input"/>
<int-aws:s3-outbound-channel-adapter
id="withCustomNameGenerator"
channel="input"
bucket="TestBucket"
accessKey="dummy"
secretKey="dummy"
remote-directory="/"
file-name-generator="customFileNameGenerator"
file-name-generation-expression="headers['name']"
auto-startup="false"/>
<bean id="customFileNameGenerator"
class="org.springframework.integration.aws.s3.config.xml.AmazonS3OutboundChannelAdapterParserTests.DummyFileNameGenerator"/>
</beans>

28
src/test/resources/s3-custom-operations-with-disallowed-attributes.xml
View File

@@ -1,28 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:integration="http://www.springframework.org/schema/integration"
xmlns:int-aws="http://www.springframework.org/schema/integration/aws"
xsi:schemaLocation=
"http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/integration/aws http://www.springframework.org/schema/integration/aws/spring-integration-aws.xsd
http://www.springframework.org/schema/integration http://www.springframework.org/schema/integration/spring-integration.xsd">
<integration:channel id="input"/>
<int-aws:s3-outbound-channel-adapter
id="withCustomNameGenerator"
channel="input"
s3-operations="customOps"
bucket="TestBucket"
accessKey="dummy"
secretKey="dummy"
multipart-upload-threshold="5120"
remote-directory="/"
file-name-generation-expression="headers['name']"
auto-startup="false"/>
<bean id="customOps"
class="org.springframework.integration.aws.s3.config.xml.AmazonS3OutboundChannelAdapterParserTests.DummyS3Operations"/>
</beans>

21
src/test/resources/s3-multiupload-lessthan-5120.xml
View File

@@ -1,21 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:integration="http://www.springframework.org/schema/integration"
xmlns:int-aws="http://www.springframework.org/schema/integration/aws"
xsi:schemaLocation=
"http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/integration/aws http://www.springframework.org/schema/integration/aws/spring-integration-aws.xsd
http://www.springframework.org/schema/integration http://www.springframework.org/schema/integration/spring-integration.xsd">
<integration:channel id="input"/>
<int-aws:s3-outbound-channel-adapter
id="withDefaultServices"
channel="input"
bucket="TestBucket"
accessKey="dummy"
secretKey="dummy"
remote-directory-expression="headers['remoteDirectory']"
multipart-upload-threshold="1024"/>
</beans>

72
src/test/resources/s3-valid-inbound-cases.xml
View File

@@ -1,72 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:integration="http://www.springframework.org/schema/integration"
xmlns:int-aws="http://www.springframework.org/schema/integration/aws"
xmlns:context="http://www.springframework.org/schema/context"
xmlns:util="http://www.springframework.org/schema/util"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/integration http://www.springframework.org/schema/integration/spring-integration.xsd
http://www.springframework.org/schema/context http://www.springframework.org/schema/context/spring-context.xsd
http://www.springframework.org/schema/integration/aws http://www.springframework.org/schema/integration/aws/spring-integration-aws.xsd
http://www.springframework.org/schema/util http://www.springframework.org/schema/util/spring-util.xsd">
<util:properties id="placeholderProperties"/>
<context:property-placeholder properties-ref="placeholderProperties"/>
<bean id="localDirectory" class="java.lang.String">
<constructor-arg value="#{T(java.lang.System).getProperty('java.io.tmpdir')}"/>
</bean>
<integration:channel id="inbound"/>
<int-aws:s3-inbound-channel-adapter
id="validInbound"
bucket="TestBucket"
propertiesFile="testawscredentials.properties"
temporary-suffix=".temp"
local-directory="${foo.dir:/}"
remote-directory="remote"
accept-sub-folders="true"
max-objects-per-batch="100"
file-name-regex="[A-Za-z0-9]+\\.txt"
channel="inbound"
auto-startup="false">
<integration:poller fixed-rate="1000"/>
</int-aws:s3-inbound-channel-adapter>
<int-aws:s3-inbound-channel-adapter
id="withAWSEndpoint"
bucket="TestBucket"
propertiesFile="testawscredentials.properties"
aws-endpoint="s3-eu-west-1.amazonaws.com"
temporary-suffix=".temp"
local-directory-expression="@localDirectory"
remote-directory="remote"
accept-sub-folders="true"
channel="inbound"
auto-startup="false">
<integration:poller fixed-rate="1000"/>
</int-aws:s3-inbound-channel-adapter>
<int-aws:s3-inbound-channel-adapter
id="validInboundWithCustomOps"
bucket="TestBucket"
s3-operations="customOps"
propertiesFile="testawscredentials.properties"
temporary-suffix=".temp"
local-directory-expression="@localDirectory"
remote-directory="remote"
accept-sub-folders="true"
max-objects-per-batch="100"
file-name-regex="[A-Za-z0-9]+\\.txt"
channel="inbound"
auto-startup="false">
<integration:poller fixed-rate="1000"/>
</int-aws:s3-inbound-channel-adapter>
<bean id="customOps"
class="org.springframework.integration.aws.s3.config.xml.AmazonS3DummyOperations"/>
</beans>

104
src/test/resources/s3-valid-outbound-cases.xml
View File

@@ -1,104 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:integration="http://www.springframework.org/schema/integration"
xmlns:int-aws="http://www.springframework.org/schema/integration/aws"
xsi:schemaLocation=
"http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/integration/aws http://www.springframework.org/schema/integration/aws/spring-integration-aws.xsd
http://www.springframework.org/schema/integration http://www.springframework.org/schema/integration/spring-integration.xsd">
<integration:channel id="input"/>
<int-aws:s3-outbound-channel-adapter
id="withCustomService"
channel="input"
bucket="TestBucket"
accessKey="dummy"
secretKey="dummy"
s3-operations="customOps"
remote-directory="/"
auto-startup="false"/>
<int-aws:s3-outbound-channel-adapter
id="withDefaultServices"
channel="input"
bucket="TestBucket"
accessKey="dummy"
secretKey="dummy"
remote-directory-expression="headers['remoteDirectory']"
charset="US-ASCII"
thread-pool-executor="executor"
multipart-upload-threshold="5120"
temporary-suffix=".write"
file-name-generation-expression="headers['name']"/>
<int-aws:s3-outbound-channel-adapter
id="withCustomNameGenerator"
channel="input"
bucket="TestBucket"
accessKey="dummy"
secretKey="dummy"
remote-directory="/"
file-name-generator="customFileNameGenerator"
auto-startup="false"/>
<int-aws:s3-outbound-channel-adapter
id="withCustomEndpoint"
aws-endpoint="http://s3-eu-west-1.amazonaws.com"
channel="input"
bucket="TestBucket"
accessKey="dummy"
secretKey="dummy"
remote-directory="/"
auto-startup="false"/>
<int-aws:s3-outbound-channel-adapter
id="withHandlerChain"
channel="input"
bucket="TestBucket"
remote-directory="/"
charset="US-ASCII"
s3-operations="customOps"
file-name-generation-expression="headers['name']"
propertiesFile="testawscredentials.properties">
<int-aws:request-handler-advice-chain>
<bean class="org.springframework.integration.aws.s3.config.xml.AmazonS3OutboundChannelAdapterParserTests$FooAdvice"/>
</int-aws:request-handler-advice-chain>
</int-aws:s3-outbound-channel-adapter>
<int-aws:s3-outbound-channel-adapter
id="withChannelAdapterAttributes"
channel="input"
bucket="TestBucket"
accessKey="dummy"
secretKey="dummy"
remote-directory="/"
charset="US-ASCII"
phase="100"
auto-startup="false"
thread-pool-executor="executor"
multipart-upload-threshold="5120"
temporary-suffix=".write"
file-name-generation-expression="headers['name']"/>
<bean id="queue" class="java.util.concurrent.ArrayBlockingQueue">
<constructor-arg value="10"/>
</bean>
<bean id="executor" class="java.util.concurrent.ThreadPoolExecutor">
<constructor-arg index="0" value="5"/>
<constructor-arg index="1" value="10"/>
<constructor-arg index="2" value="10"/>
<constructor-arg index="3" value="MILLISECONDS"/>
<constructor-arg index="4" ref="queue"/>
</bean>
<bean id="customOps"
class="org.springframework.integration.aws.s3.config.xml.AmazonS3DummyOperations"/>
<bean id="customFileNameGenerator"
class="org.springframework.integration.aws.s3.config.xml.AmazonS3OutboundChannelAdapterParserTests.DummyFileNameGenerator"/>
</beans>

26
src/test/resources/s3-with-none-of-direxpr-and-dir.xml
View File

@@ -1,26 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:integration="http://www.springframework.org/schema/integration"
xmlns:int-aws="http://www.springframework.org/schema/integration/aws"
xsi:schemaLocation=
"http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/integration/aws http://www.springframework.org/schema/integration/aws/spring-integration-aws.xsd
http://www.springframework.org/schema/integration http://www.springframework.org/schema/integration/spring-integration.xsd">
<integration:channel id="inbound"/>
<int-aws:s3-inbound-channel-adapter
id="validInbound"
bucket="TestBucket"
propertiesFile="testawscredentials.properties"
temporary-suffix=".temp"
remote-directory="remote"
accept-sub-folders="true"
max-objects-per-batch="100"
file-name-regex="[A-Za-z0-9]+\\.txt"
channel="inbound">
<integration:poller fixed-rate="1000"/>
</int-aws:s3-inbound-channel-adapter>
</beans>

2
src/test/resources/testawscredentials.properties
View File

@@ -1,2 +0,0 @@
accessKey=dummy
secretKey=dummy