Spring/spring-integration-aws
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add S3 MessageSource implementation

Browse Source 
JIRA: https://jira.spring.io/browse/INTEXT-51,
https://jira.spring.io/browse/INTEXT-194,
https://jira.spring.io/browse/INTEXT-199

* Add S3 Inbound Channel Adapter fully based on the `AbstractInboundFileSynchronizer`,
`AbstractInboundFileSynchronizingMessageSource` implementation.
* Provide the standard SI `RemoteFileTemplate` and `SessionFactory` abstractions implementations.
* Upgrade to Gradle-2.12
* Provide Namespace support
* Ensure everything with tests
* Remove legacy, redundant implementation, infrastructure code for it and test-cases.
Everything in favor of the new implementation.
This commit is contained in:
Artem Bilan
2016-04-04 19:59:02 -04:00
parent 5167b36b4d
commit 29e255cc64
86 changed files with 1283 additions and 7779 deletions
Download Patch File Download Diff File Expand all files Collapse all files

60
src/test/java/org/springframework/integration/aws/common/AWSClientFactoryTest.java
View File

@@ -1,60 +0,0 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.aws.common;
import org.junit.Assert;
import org.junit.Test;
import org.springframework.integration.aws.core.AbstractAWSClientFactory;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.services.sqs.AmazonSQSClient;
/**
* The test class for {@link AbstractAmazonWSClientFactory}
*
* @author Amol Nayak
*
* @since 0.5
*
*/
public class AWSClientFactoryTest {
AbstractAWSClientFactory<AmazonSQSClient> factory = new AbstractAWSClientFactory<AmazonSQSClient>() {
@Override
protected AmazonSQSClient getClientImplementation() {
return new AmazonSQSClient((AWSCredentials)null);
}
};
@Test
public void getRegionSpecificEndpoints() {
//TODO: Use Spring integration test to inspect and assert the variables within
AmazonSQSClient usEastClient = factory.getClient("https://queue.amazonaws.com/123456789012/MyTestQueue");
Assert.assertNotNull(usEastClient);
Assert.assertEquals(factory.getClientMap().size(),1);
AmazonSQSClient usEastClient1 = factory.getClient("https://queue.amazonaws.com/123456789012/MyTestQueue1");
Assert.assertNotNull(usEastClient1);
Assert.assertEquals(usEastClient,usEastClient1);
Assert.assertEquals(factory.getClientMap().size(),1);
AmazonSQSClient apacClient1 = factory.getClient("https://ap-southeast-1.queue.amazonaws.com/123456789012/MyApacTestQueue");
Assert.assertNotNull(apacClient1);
Assert.assertEquals(factory.getClientMap().size(),2);
}
}

165
src/test/java/org/springframework/integration/aws/common/AWSTestUtils.java
View File

@@ -1,165 +0,0 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.aws.common;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.UnsupportedEncodingException;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.util.ArrayList;
import java.util.List;
import org.junit.Assert;
import org.apache.commons.codec.binary.Base64;
import org.springframework.integration.aws.core.AWSCredentials;
import org.springframework.integration.aws.core.PropertiesAWSCredentials;
/**
* Common test class utility to be used by AmazonWS test cases
*
* @author Amol Nayak
*
* @since 0.5
*
*/
public final class AWSTestUtils {
private static PropertiesAWSCredentials credentials;
private AWSTestUtils() {
throw new AssertionError("Cannot instantiate a utility class");
}
/**
* Method that will be used to test the contents of the file to assert we are getting the
* the right value
*
* @param permFile
* @param expectedContent
* @throws IOException
*/
public static final void assertFileContent(File permFile, String expectedContent)
throws IOException {
BufferedReader reader = null;
try {
reader = new BufferedReader(new InputStreamReader(new FileInputStream(permFile)));
Assert.assertEquals(expectedContent, reader.readLine());
} finally {
if(reader != null) {
reader.close();
}
}
}
/**
* Writes the content to the file at given location.
*
* @param path
* @param content
*/
public static final void writeToFile(String path, String content) throws IOException {
//create the required directories if needed
if(path.contains(File.separator)) {
int index = path.lastIndexOf(File.separatorChar);
if(index != 0) {
new File(path.substring(0, index)).mkdirs();
}
}
File file = new File(path);
FileOutputStream fos = new FileOutputStream(file);
fos.write(content.getBytes());
fos.close();
}
/**
* The static helper method that would be used by other AWS tests to
* get the implementation of the {@link AWSCredentials} instance
* @throws Exception
*/
public static AWSCredentials getCredentials() {
if(credentials == null) {
credentials =
new PropertiesAWSCredentials("classpath:awscredentials.properties");
try {
credentials.afterPropertiesSet();
} catch (Exception e) {
return null;
}
}
return credentials;
}
/**
* When passed a {@link File} instance for the root directory, the contents are recursively
* checked and the listing of the directories is retrieved.
*
* @param rootDirectory
*/
public static List<File> getContentsRecursively(File rootDirectory) {
if(!rootDirectory.isDirectory()) {
return null;
}
else {
List<File> files = new ArrayList<File>();
getContentsRecursively(rootDirectory, files);
return files;
}
}
/**
* Recursively gets all the files present under the provided directory
* @param file
* @param files
*/
private static void getContentsRecursively(File file, List<File> files) {
if(file.isFile()) {
files.add(file);
}
else if(file.isDirectory()){
File[] children = file.listFiles();
if(children != null && children.length != 0) {
for(File child:children) {
getContentsRecursively(child, files);
}
}
}
}
/**
* Helper method that will be used to generate the base64 encoded MD5 hash of the string
*
* @param input
*/
public static String md5Hash(String input) {
try {
MessageDigest digest = MessageDigest.getInstance("MD5");
byte[] digestedBytes = digest.digest(input.getBytes("UTF-8"));
return new String(Base64.encodeBase64(digestedBytes),"UTF-8");
} catch (NoSuchAlgorithmException e) {
e.printStackTrace();
return null;
} catch(UnsupportedEncodingException e) {
e.printStackTrace();
return null;
}
}
}

51
src/test/java/org/springframework/integration/aws/common/BaseTestCase.java
View File

@@ -1,51 +0,0 @@
/*
* Copyright 2002-2012 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.aws.common;
import java.io.IOException;
import java.util.Properties;
/**
* Class extended by some of the test cases which are not spring based to get access to the properties
* and other common functionality
*
* @author Amol Nayak
*
* @since 0.5
*
*/
public abstract class BaseTestCase {
protected static String propsLocation = "testprops.properties";
protected static Properties props;
static {
props = new Properties();
try {
props.load(ClassLoader.getSystemClassLoader().getResourceAsStream(propsLocation));
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
protected static String getProperty(String key) {
if(props != null)
return props.getProperty(key);
else
return null;
}
}

134
src/test/java/org/springframework/integration/aws/config/xml/AbstractAWSOutboundChannelAdapterParserTests.java
View File

@@ -1,134 +0,0 @@
/*
* Copyright 2002-2014 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.aws.config.xml;
import org.junit.Assert;
import org.junit.AfterClass;
import org.junit.Before;
import org.junit.Test;
import org.springframework.context.ApplicationContext;
import org.springframework.context.support.ClassPathXmlApplicationContext;
import org.springframework.integration.aws.core.AWSCredentials;
import org.springframework.integration.endpoint.AbstractEndpoint;
import org.springframework.integration.test.util.TestUtils;
import org.springframework.messaging.MessageHandler;
/**
* The Abstract test class for the AWS outbound adapter parsers
*
* @author Amol Nayak
* @author Rob Harrop
*
* @since 0.5
*
*/
public abstract class AbstractAWSOutboundChannelAdapterParserTests<T extends MessageHandler> {
protected static ClassPathXmlApplicationContext ctx;
private static boolean isInitialized;
/**
* The bean name expected to be present in the context which would be used when the
* element is defined with the propertiesFile attribute
*/
private final String CREDENTIALS_TEST_ONE = "credentialsTestOne";
/**
* The bean name expected to be present in the context which would be used when the
* element is defined with the accessKey and the secretKey definition
*/
private final String CREDENTIALS_TEST_TWO = "credentialsTestTwo";
@Before
public void setup() {
if(!isInitialized) {
String contextLocation = getConfigFilePath();
Assert.assertNotNull("Non null path value expected", contextLocation);
ctx = new ClassPathXmlApplicationContext(contextLocation);
Assert.assertTrue("Bean with id " + CREDENTIALS_TEST_ONE
+ " expected to be present in the context for credentials test",
ctx.containsBean(CREDENTIALS_TEST_ONE));
Assert.assertTrue("Bean with id " + CREDENTIALS_TEST_TWO
+ " expected to be present in the context for credentials test",
ctx.containsBean(CREDENTIALS_TEST_TWO));
isInitialized = true;
}
}
@AfterClass
public static void destroy() {
ctx.close();
}
/**
* Gets the Message handler implementation for the given bean id
* @param beanId
*/
@SuppressWarnings("unchecked")
protected T getMessageHandlerForBeanDefinition(String beanId) {
AbstractEndpoint endpoint = (AbstractEndpoint)ctx.getBean(beanId);
return (T)TestUtils.getPropertyValue(endpoint, "handler");
}
/**
* Gets the config file path that would be used to create the {@link ApplicationContext} instance
* sub class should implement this method to return a value of the config to be used to create the
* context
*
*/
protected abstract String getConfigFilePath();
/**
* The subclass should return the {@link AWSCredentials} instance that is being used
* to configure the adapters
*
*/
protected abstract AWSCredentials getCredentials();
/**
* The test class for the AWS Credentials test that used property files
*/
@Test
public final void awsCredentialsTestWithPropFiles() {
MessageHandler handler = getMessageHandlerForBeanDefinition(CREDENTIALS_TEST_ONE);
AWSCredentials credentials = getCredentials();
String accessKey = credentials.getAccessKey();
String secretKey = credentials.getSecretKey();
AWSCredentials configuredCredentials =
TestUtils.getPropertyValue(handler, "credentials",AWSCredentials.class);
Assert.assertEquals(accessKey, configuredCredentials.getAccessKey());
Assert.assertEquals(secretKey, configuredCredentials.getSecretKey());
}
/**
* The test class for the AWS Credentials test that used accessKey and secretKey elements
*/
@Test
public final void awsCredentialsTestWithoutPropFiles() {
MessageHandler handler = getMessageHandlerForBeanDefinition(CREDENTIALS_TEST_TWO);
AWSCredentials credentials = getCredentials();
String accessKey = credentials.getAccessKey();
String secretKey = credentials.getSecretKey();
AWSCredentials configuredCredentials =
TestUtils.getPropertyValue(handler, "credentials",AWSCredentials.class);
Assert.assertEquals(accessKey, configuredCredentials.getAccessKey());
Assert.assertEquals(secretKey, configuredCredentials.getSecretKey());
}
}

49
src/test/java/org/springframework/integration/aws/config/xml/S3InboundChannelAdapterParserTests-context.xml Normal file
View File

@@ -0,0 +1,49 @@
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:int-aws="http://www.springframework.org/schema/integration/aws"
xmlns:int="http://www.springframework.org/schema/integration"
xsi:schemaLocation="http://www.springframework.org/schema/beans http://www.springframework.org/schema/beans/spring-beans.xsd
http://www.springframework.org/schema/integration/aws http://www.springframework.org/schema/integration/aws/spring-integration-aws.xsd
http://www.springframework.org/schema/integration http://www.springframework.org/schema/integration/spring-integration.xsd">
<bean id="s3" class="org.mockito.Mockito" factory-method="mock">
<constructor-arg value="com.amazonaws.services.s3.AmazonS3"/>
</bean>
<bean id="s3SessionFactory" class="org.springframework.integration.aws.support.S3SessionFactory">
<constructor-arg ref="s3"/>
</bean>
<bean id="comparator" class="org.mockito.Mockito" factory-method="mock">
<constructor-arg value="java.util.Comparator"/>
</bean>
<int:channel id="s3Channel">
<int:queue/>
</int:channel>
<bean id="fooString" class="java.lang.String">
<constructor-arg value="foo"/>
</bean>
<bean id="acceptAllFilter" class="org.springframework.integration.file.filters.AcceptAllFileListFilter"/>
<int-aws:s3-inbound-channel-adapter id="s3Inbound"
channel="s3Channel"
session-factory="s3SessionFactory"
auto-create-local-directory="true"
auto-startup="false"
delete-remote-files="true"
preserve-timestamp="true"
filename-pattern="*.txt"
local-directory="."
local-filename-generator-expression="#this.toUpperCase() + '.a' + @fooString"
comparator="comparator"
temporary-file-suffix=".foo"
local-filter="acceptAllFilter"
remote-directory-expression="'foo/bar'">
<int:poller fixed-rate="1000"/>
</int-aws:s3-inbound-channel-adapter>
</beans>

117
src/test/java/org/springframework/integration/aws/config/xml/S3InboundChannelAdapterParserTests.java Normal file
View File

@@ -0,0 +1,117 @@
/*
* Copyright 2016 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.aws.config.xml;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertTrue;
import java.lang.reflect.Method;
import java.util.Collection;
import java.util.Comparator;
import java.util.concurrent.PriorityBlockingQueue;
import java.util.concurrent.atomic.AtomicReference;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.expression.Expression;
import org.springframework.integration.aws.inbound.S3InboundFileSynchronizer;
import org.springframework.integration.aws.inbound.S3InboundFileSynchronizingMessageSource;
import org.springframework.integration.aws.support.filters.S3SimplePatternFileListFilter;
import org.springframework.integration.endpoint.SourcePollingChannelAdapter;
import org.springframework.integration.file.filters.AcceptAllFileListFilter;
import org.springframework.integration.file.remote.session.SessionFactory;
import org.springframework.integration.file.remote.synchronizer.AbstractInboundFileSynchronizer;
import org.springframework.integration.test.util.TestUtils;
import org.springframework.messaging.MessageChannel;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import org.springframework.util.ReflectionUtils;
/**
* @author Artem Bilan
*/
@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration
public class S3InboundChannelAdapterParserTests {
@Autowired
private SourcePollingChannelAdapter s3Inbound;
@Autowired
private Comparator<?> comparator;
@Autowired
private MessageChannel s3Channel;
@Autowired
private AcceptAllFileListFilter<?> acceptAllFilter;
@Autowired
private SessionFactory<?> s3SessionFactory;
@Test
public void testFtpInboundChannelAdapterComplete() throws Exception {
assertFalse(TestUtils.getPropertyValue(this.s3Inbound, "autoStartup", Boolean.class));
PriorityBlockingQueue<?> blockingQueue = TestUtils.getPropertyValue(this.s3Inbound,
"source.fileSource.toBeReceived", PriorityBlockingQueue.class);
Comparator<?> comparator = blockingQueue.comparator();
assertSame(this.comparator, comparator);
assertEquals("s3Inbound", this.s3Inbound.getComponentName());
assertEquals("aws:s3-inbound-channel-adapter", this.s3Inbound.getComponentType());
assertSame(this.s3Channel, TestUtils.getPropertyValue(this.s3Inbound, "outputChannel"));
S3InboundFileSynchronizingMessageSource inbound = TestUtils.getPropertyValue(this.s3Inbound, "source",
S3InboundFileSynchronizingMessageSource.class);
S3InboundFileSynchronizer fisync = TestUtils.getPropertyValue(inbound, "synchronizer",
S3InboundFileSynchronizer.class);
assertEquals("'foo/bar'",
TestUtils.getPropertyValue(fisync, "remoteDirectoryExpression", Expression.class)
.getExpressionString());
assertNotNull(TestUtils.getPropertyValue(fisync, "localFilenameGeneratorExpression"));
assertTrue(TestUtils.getPropertyValue(fisync, "preserveTimestamp", Boolean.class));
assertEquals(".foo", TestUtils.getPropertyValue(fisync, "temporaryFileSuffix", String.class));
String remoteFileSeparator = (String) TestUtils.getPropertyValue(fisync, "remoteFileSeparator");
assertNotNull(remoteFileSeparator);
assertEquals("", remoteFileSeparator);
S3SimplePatternFileListFilter filter = TestUtils.getPropertyValue(fisync, "filter",
S3SimplePatternFileListFilter.class);
assertNotNull(filter);
assertSame(this.s3SessionFactory, TestUtils.getPropertyValue(fisync, "remoteFileTemplate.sessionFactory"));
assertTrue(TestUtils.getPropertyValue(inbound, "fileSource.scanner.filter.fileFilters", Collection.class)
.contains(this.acceptAllFilter));
final AtomicReference<Method> genMethod = new AtomicReference<Method>();
ReflectionUtils.doWithMethods(AbstractInboundFileSynchronizer.class, new ReflectionUtils.MethodCallback() {
@Override
public void doWith(Method method) throws IllegalArgumentException, IllegalAccessException {
if ("generateLocalFileName".equals(method.getName())) {
method.setAccessible(true);
genMethod.set(method);
}
}
});
assertEquals("FOO.afoo", genMethod.get().invoke(fisync, "foo"));
}
}

152
src/test/java/org/springframework/integration/aws/core/AbstractAWSClientFactoryTests.java
View File

@@ -1,152 +0,0 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.aws.core;
import java.net.URI;
import java.util.Map;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.springframework.integration.test.util.TestUtils;
import com.amazonaws.AmazonWebServiceClient;
/**
* The abstract test class for Amazon WebService client factory tests
* @author Amol Nayak
*
* @since 0.5
*
*/
public abstract class AbstractAWSClientFactoryTests<T extends AmazonWebServiceClient> {
protected AbstractAWSClientFactory<T> factory;
@Before
public final void setup() {
factory = getFactory();
}
/**
* The subclass is responsible for returning the appropriate factory implementation
*/
protected abstract AbstractAWSClientFactory<T> getFactory();
/**
* Gets the endpoint for the service in US-EAST-1 region
*/
protected abstract String getUSEast1Endpoint();
/**
* Gets the endpoint for the service in US-EAST-1 region
*/
protected abstract String getEUWest1Endpoint();
/**
* Gets the Suffix for the endpoint URL which is service specific
*/
protected abstract String getSuffix();
/**
* The test case for giving a US east endpoint without protocol
*/
@Test
public void withUSEast1EndpointWithoutProtocol() {
String usEast1 = getUSEast1Endpoint();
factory.clear();
T client = factory.getClient(usEast1 + getSuffix());
Map<String, T> map = factory.getClientMap();
Assert.assertNotNull(map);
Assert.assertEquals(1, map.size());
Assert.assertTrue("Expected one key with value https://" + usEast1, map.containsKey("https://" + usEast1));
URI endpoint = TestUtils.getPropertyValue(client, "endpoint", URI.class);
Assert.assertNotNull(endpoint);
Assert.assertEquals(usEast1,endpoint.getHost());
Assert.assertEquals("https",endpoint.getScheme());
}
/**
* Tests the factory by providing an endpoint in US east with protocol as http
*/
@Test
public void withUSEast1EndpointWithProtocol() {
String usEast1 = getUSEast1Endpoint();
factory.clear();
T client = factory.getClient("http://" + usEast1 + getSuffix());
Map<String, T> map = factory.getClientMap();
Assert.assertNotNull(map);
Assert.assertEquals(1, map.size());
Assert.assertTrue("Expected one key with value http://" + usEast1, map.containsKey("http://" + usEast1));
URI endpoint = TestUtils.getPropertyValue(client, "endpoint", URI.class);
Assert.assertNotNull(endpoint);
Assert.assertEquals(usEast1,endpoint.getHost());
Assert.assertEquals("http",endpoint.getScheme());
}
/**
* Calls the getClient multiple times to get the same client instance on each invocation
*/
@Test
public void withMultipleCallsToSameEndpoint() {
String usEast1 = getUSEast1Endpoint();
factory.clear();
T client = factory.getClient("https://" + usEast1 + getSuffix());
Map<String, T> map = factory.getClientMap();
Assert.assertNotNull(map);
Assert.assertEquals(1, map.size());
Assert.assertTrue("Expected one key with value http://" + usEast1, map.containsKey("https://" + usEast1));
//default to https
T client1 = factory.getClient(usEast1 + getSuffix());
map = factory.getClientMap();
Assert.assertEquals(1, map.size());
Assert.assertTrue("Expected one key with value http://" + usEast1, map.containsKey("https://" + usEast1));
Assert.assertTrue("Expecting to get the same instance of the client, but was not", client == client1);
}
/**
*Calls to different endpoints on the same client, expected to return a client with
*appropriate endpoint URI set
*/
@Test
public void withMultipleCallsToDifferentEndpoints() {
String usEast1 = getUSEast1Endpoint();
String euWest1 = getEUWest1Endpoint();
factory.clear();
T client = factory.getClient(usEast1 + getSuffix());
Map<String, T> map = factory.getClientMap();
Assert.assertNotNull(map);
Assert.assertEquals(1, map.size());
Assert.assertTrue("Expected one key with value https://" + usEast1, map.containsKey("https://" + usEast1));
URI endpoint = TestUtils.getPropertyValue(client, "endpoint", URI.class);
Assert.assertNotNull(endpoint);
Assert.assertEquals(usEast1,endpoint.getHost());
Assert.assertEquals("https",endpoint.getScheme());
client = factory.getClient(euWest1 + getSuffix());
map = factory.getClientMap();
Assert.assertNotNull(map);
Assert.assertEquals(2, map.size());
Assert.assertTrue("Expected one key with value https://" + euWest1, map.containsKey("https://" + euWest1));
endpoint = TestUtils.getPropertyValue(client, "endpoint", URI.class);
Assert.assertNotNull(endpoint);
Assert.assertEquals(euWest1,endpoint.getHost());
Assert.assertEquals("https",endpoint.getScheme());
}
}

225
src/test/java/org/springframework/integration/aws/inbound/S3InboundChannelAdapterTests.java Normal file
View File

@@ -0,0 +1,225 @@
/*
* Copyright 2016 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.aws.inbound;
import static org.hamcrest.Matchers.instanceOf;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import static org.mockito.BDDMockito.willAnswer;
import static org.mockito.Matchers.any;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Calendar;
import java.util.List;
import org.hamcrest.Matchers;
import org.junit.BeforeClass;
import org.junit.ClassRule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.expression.Expression;
import org.springframework.expression.ExpressionParser;
import org.springframework.expression.spel.standard.SpelExpressionParser;
import org.springframework.integration.annotation.InboundChannelAdapter;
import org.springframework.integration.annotation.Poller;
import org.springframework.integration.aws.support.filters.S3RegexPatternFileListFilter;
import org.springframework.integration.channel.QueueChannel;
import org.springframework.integration.config.EnableIntegration;
import org.springframework.integration.file.filters.AcceptOnceFileListFilter;
import org.springframework.messaging.Message;
import org.springframework.messaging.PollableChannel;
import org.springframework.test.annotation.DirtiesContext;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringJUnit4ClassRunner;
import org.springframework.util.FileCopyUtils;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.model.ListObjectsRequest;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectSummary;
/**
* @author Artem Bilan
*/
@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration
@DirtiesContext
public class S3InboundChannelAdapterTests {
private static final ExpressionParser PARSER = new SpelExpressionParser();
private static final String S3_BUCKET = "S3_BUCKET";
@ClassRule
public static final TemporaryFolder TEMPORARY_FOLDER = new TemporaryFolder();
private static File REMOTE_FOLDER;
private static List<S3Object> S3_OBJECTS;
private static File LOCAL_FOLDER;
@Autowired
private PollableChannel s3FilesChannel;
@BeforeClass
public static void setup() throws IOException {
REMOTE_FOLDER = TEMPORARY_FOLDER.newFolder("remote");
File aFile = new File(REMOTE_FOLDER, "a.test");
FileCopyUtils.copy("Hello".getBytes(), aFile);
File bFile = new File(REMOTE_FOLDER, "b.test");
FileCopyUtils.copy("Bye".getBytes(), bFile);
File otherFile = new File(REMOTE_FOLDER, "otherFile");
FileCopyUtils.copy("Other".getBytes(), otherFile);
S3_OBJECTS = new ArrayList<>();
for (File file : REMOTE_FOLDER.listFiles()) {
S3Object s3Object = new S3Object();
s3Object.setBucketName(S3_BUCKET);
s3Object.setKey(file.getName());
s3Object.setObjectContent(new FileInputStream(file));
S3_OBJECTS.add(s3Object);
}
LOCAL_FOLDER = new File(TEMPORARY_FOLDER.getRoot(), "local");
}
@Test
public void testS3InboundChannelAdapter() throws IOException {
Message<?> message = this.s3FilesChannel.receive(10000);
assertNotNull(message);
assertThat(message.getPayload(), instanceOf(File.class));
File localFile = (File) message.getPayload();
assertEquals("A.TEST.a", localFile.getName());
// The test remote files are created with the current timestamp + 1 day.
assertThat(localFile.lastModified(), Matchers.greaterThan(System.currentTimeMillis()));
message = this.s3FilesChannel.receive(10000);
assertNotNull(message);
assertThat(message.getPayload(), instanceOf(File.class));
localFile = (File) message.getPayload();
assertEquals("B.TEST.a", localFile.getName());
assertThat(localFile.lastModified(), Matchers.greaterThan(System.currentTimeMillis()));
assertNull(this.s3FilesChannel.receive(10));
File file = new File(LOCAL_FOLDER, "A.TEST.a");
assertTrue(file.exists());
String content = FileCopyUtils.copyToString(new FileReader(file));
assertEquals("Hello", content);
file = new File(LOCAL_FOLDER, "B.TEST.a");
assertTrue(file.exists());
content = FileCopyUtils.copyToString(new FileReader(file));
assertEquals("Bye", content);
assertFalse(new File(LOCAL_FOLDER, "otherFile.a").exists());
}
@Configuration
@EnableIntegration
public static class Config {
@Bean
public AmazonS3 amazonS3() {
AmazonS3 amazonS3 = Mockito.mock(AmazonS3.class);
willAnswer(new Answer<ObjectListing>() {
@Override
public ObjectListing answer(InvocationOnMock invocation) throws Throwable {
ObjectListing objectListing = new ObjectListing();
List<S3ObjectSummary> objectSummaries = objectListing.getObjectSummaries();
for (S3Object s3Object : S3_OBJECTS) {
S3ObjectSummary s3ObjectSummary = new S3ObjectSummary();
s3ObjectSummary.setBucketName(S3_BUCKET);
s3ObjectSummary.setKey(s3Object.getKey());
Calendar calendar = Calendar.getInstance();
calendar.add(Calendar.DATE, 1);
s3ObjectSummary.setLastModified(calendar.getTime());
objectSummaries.add(s3ObjectSummary);
}
return objectListing;
}
}).given(amazonS3).listObjects(any(ListObjectsRequest.class));
for (final S3Object s3Object : S3_OBJECTS) {
willAnswer(new Answer<S3Object>() {
@Override
public S3Object answer(InvocationOnMock invocation) throws Throwable {
return s3Object;
}
}).given(amazonS3).getObject(S3_BUCKET, s3Object.getKey());
}
return amazonS3;
}
@Bean
public S3InboundFileSynchronizer s3InboundFileSynchronizer() {
S3InboundFileSynchronizer synchronizer = new S3InboundFileSynchronizer(amazonS3());
synchronizer.setDeleteRemoteFiles(true);
synchronizer.setPreserveTimestamp(true);
synchronizer.setRemoteDirectory(S3_BUCKET);
synchronizer.setFilter(new S3RegexPatternFileListFilter(".*\\.test$"));
Expression expression = PARSER.parseExpression("#this.toUpperCase() + '.a'");
synchronizer.setLocalFilenameGeneratorExpression(expression);
return synchronizer;
}
@Bean
@InboundChannelAdapter(value = "s3FilesChannel", poller = @Poller(fixedDelay = "100"))
public S3InboundFileSynchronizingMessageSource s3InboundFileSynchronizingMessageSource() {
S3InboundFileSynchronizingMessageSource messageSource =
new S3InboundFileSynchronizingMessageSource(s3InboundFileSynchronizer());
messageSource.setAutoCreateLocalDirectory(true);
messageSource.setLocalDirectory(LOCAL_FOLDER);
messageSource.setLocalFilter(new AcceptOnceFileListFilter<File>());
return messageSource;
}
@Bean
public PollableChannel s3FilesChannel() {
return new QueueChannel();
}
}
}

273
src/test/java/org/springframework/integration/aws/s3/AmazonS3InboundSynchronizationMessageSourceTests.java
View File

@@ -1,273 +0,0 @@
/*
* Copyright 2002-2014 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.aws.s3;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
import static org.springframework.integration.aws.common.AWSTestUtils.md5Hash;
import static org.springframework.integration.aws.s3.AmazonS3OperationsMockingUtil.mockAmazonS3Operations;
import static org.springframework.integration.aws.s3.AmazonS3OperationsMockingUtil.mockS3Operations;
import static org.springframework.integration.test.util.TestUtils.getPropertyValue;
import java.io.File;
import java.io.IOException;
import java.util.Arrays;
import java.util.Map;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.mockito.Mockito;
import org.springframework.beans.factory.BeanFactory;
import org.springframework.expression.common.LiteralExpression;
import org.springframework.integration.aws.core.BasicAWSCredentials;
import org.springframework.integration.aws.s3.core.AbstractAmazonS3Operations;
import org.springframework.integration.aws.s3.core.AmazonS3Object;
import org.springframework.integration.aws.s3.core.AmazonS3ObjectACL;
import org.springframework.integration.aws.s3.core.AmazonS3Operations;
import org.springframework.integration.aws.s3.core.PaginatedObjectsView;
import org.springframework.messaging.Message;
/**
* The test class for {@link AmazonS3InboundSynchronizationMessageSource}
*
* @author Amol Nayak
* @author Rob Harrop
*
* @since 0.5
*
*/
public class AmazonS3InboundSynchronizationMessageSourceTests {
@Rule
public TemporaryFolder temp = new TemporaryFolder();
private static AmazonS3Operations operations;
@BeforeClass
public static void setup() {
operations = mockS3Operations();
}
/**
* Tests by providing null credentials.
*/
@Test(expected = IllegalArgumentException.class)
public void withNullCredentials() {
AmazonS3InboundSynchronizationMessageSource src = new AmazonS3InboundSynchronizationMessageSource();
src.setCredentials(null);
}
/**
* Tests by providing null temporary suffix
*/
@Test(expected = IllegalArgumentException.class)
public void withNullTempSuffix() {
AmazonS3InboundSynchronizationMessageSource src = new AmazonS3InboundSynchronizationMessageSource();
src.setTemporarySuffix(null);
}
/**
* Tests by providing null wildcard
*/
@Test(expected = IllegalArgumentException.class)
public void withNullWildcard() {
AmazonS3InboundSynchronizationMessageSource src = new AmazonS3InboundSynchronizationMessageSource();
src.setFileNameWildcard(null);
}
/**
* Tests by providing null regex
*/
@Test(expected = IllegalArgumentException.class)
public void withNullRegex() {
AmazonS3InboundSynchronizationMessageSource src = new AmazonS3InboundSynchronizationMessageSource();
src.setFileNameRegex(null);
}
/**
* Tests by providing both regex and wildcard
*/
@Test(expected = IllegalArgumentException.class)
public void withBothRegexAndWildcard() {
AmazonS3InboundSynchronizationMessageSource src = new AmazonS3InboundSynchronizationMessageSource();
src.setFileNameRegex("[a-z]+\\.txt");
src.setFileNameWildcard("*.txt");
}
/**
* Tests by providing both wildcard and regex, unlike previous one, this sets the wildcard first
*/
@Test(expected = IllegalArgumentException.class)
public void withBothWildcardAndRegex() {
AmazonS3InboundSynchronizationMessageSource src = new AmazonS3InboundSynchronizationMessageSource();
src.setFileNameWildcard("*.txt");
src.setFileNameRegex("[a-z]+\\.txt");
}
/**
* Tests providing null remote directory
*/
@Test(expected = IllegalArgumentException.class)
public void withNullRemoteDirectory() {
AmazonS3InboundSynchronizationMessageSource src = new AmazonS3InboundSynchronizationMessageSource();
src.setRemoteDirectory(null);
}
/**
*Tests with a non existent local directory
*/
@Test(expected = IllegalArgumentException.class)
public void withNonExistentLocalDirectory() {
AmazonS3InboundSynchronizationMessageSource src = new AmazonS3InboundSynchronizationMessageSource();
src.setDirectory(new LiteralExpression("SomeNotExistentDir"));
src.setBeanFactory(Mockito.mock(BeanFactory.class));
src.afterPropertiesSet();
}
/**
* Tests with a {@link File} instance that is not a directory
* @throws IOException
*/
@Test(expected = IllegalArgumentException.class)
public void withNonDirectory() throws IOException {
File file = temp.newFile("SomeFile.txt");
AmazonS3InboundSynchronizationMessageSource src = new AmazonS3InboundSynchronizationMessageSource();
src.setDirectory(new LiteralExpression(file.getAbsolutePath()));
src.setBeanFactory(Mockito.mock(BeanFactory.class));
src.afterPropertiesSet();
}
/**
* Tests with a null s3 operation.
*/
@Test(expected = IllegalArgumentException.class)
public void withNullS3Operations() {
AmazonS3InboundSynchronizationMessageSource src = new AmazonS3InboundSynchronizationMessageSource();
src.setS3Operations(null);
}
/**
* Doesn't set the {@link AmazonS3Operations} instance and relies on the default one.
* sets the temp suffix and the thread pool executor.
*/
@Test
public void withDefaultS3Service() {
AmazonS3InboundSynchronizationMessageSource src = new AmazonS3InboundSynchronizationMessageSource();
BasicAWSCredentials credentials = new BasicAWSCredentials("dummy", "dummy");
src.setTemporarySuffix(".temp");
src.setCredentials(credentials);
src.setDirectory(new LiteralExpression(temp.getRoot().getAbsolutePath()));
src.setBeanFactory(Mockito.mock(BeanFactory.class));
src.afterPropertiesSet();
assertEquals(".temp", getPropertyValue(src, "s3Operations.temporaryFileSuffix", String.class));
}
/**
* Instantiates with a custom implementation of {@link AmazonS3Operations}
* which extends from {@link AbstractAmazonS3Operations}. Also sets the following
* s3Operations, directory, fileNameRegex, remoteDirectory, maxObjectsPerBatch and temporarySuffix
* attributes.
*/
@Test
public void withCustomS3Operations() {
BasicAWSCredentials credentials = new BasicAWSCredentials("dummy", "dummy");
AbstractAmazonS3Operations ops = new AbstractAmazonS3Operations(credentials) {
@Override
protected void doPut(String bucket, String key, File file,
AmazonS3ObjectACL objectACL, Map<String, String> userMetadata,
String stringContentMD5) {
}
@Override
protected PaginatedObjectsView doListObjects(String bucketName,
String nextMarker, int pageSize, String prefix) {
return null;
}
@Override
protected AmazonS3Object doGetObject(String bucketName, String key) {
return null;
}
};
AmazonS3InboundSynchronizationMessageSource src = new AmazonS3InboundSynchronizationMessageSource();
src.setS3Operations(ops);
src.setBucket("testbucket");
src.setDirectory(new LiteralExpression(temp.getRoot().getAbsolutePath()));
src.setFileNameRegex("[a-z]+\\.txt");
src.setRemoteDirectory("remotedirectory");
src.setMaxObjectsPerBatch(15);
src.setTemporarySuffix(".temp");
src.setAcceptSubFolders(true);
src.setDirectory(new LiteralExpression(temp.getRoot().getAbsolutePath()));
src.setBeanFactory(Mockito.mock(BeanFactory.class));
src.afterPropertiesSet();
assertEquals(ops, getPropertyValue(src, "s3Operations", AmazonS3Operations.class));
assertEquals(".temp", getPropertyValue(src, "s3Operations.temporaryFileSuffix", String.class));
assertEquals("testbucket", getPropertyValue(src, "bucket", String.class));
assertEquals(temp.getRoot(), getPropertyValue(src, "directory"));
assertEquals("[a-z]+\\.txt", getPropertyValue(src, "synchronizer.fileNameRegex", String.class));
assertEquals(true, getPropertyValue(src, "synchronizer.acceptSubFolders", Boolean.class));
assertEquals(15, getPropertyValue(src, "synchronizer.maxObjectsPerBatch", Integer.class).intValue());
assertEquals("remotedirectory", getPropertyValue(src, "remoteDirectory", String.class));
}
/**
* Synchronizes the local directory to a remote bucket
*/
@Test
public void synchronizeWithLocalDirectory() {
mockAmazonS3Operations(Arrays.asList(
new String[] {"test.txt", "test.txt", md5Hash("test.txt"), null},
new String[] {"sub1/test.txt", "sub1/test.txt", md5Hash("sub1/test.txt"), null},
new String[] {"sub1/sub11/test.txt", "sub1/sub11/test.txt", md5Hash("sub1/sub11/test.txt"), null},
new String[] {"sub2/test.txt", "sub2/test.txt", md5Hash("sub2/test.txt"), null}
));
AmazonS3InboundSynchronizationMessageSource src = new AmazonS3InboundSynchronizationMessageSource();
src.setS3Operations(operations);
src.setBucket(AmazonS3OperationsMockingUtil.BUCKET);
src.setDirectory(new LiteralExpression(temp.getRoot().getAbsolutePath()));
src.setFileNameRegex("[a-z]+\\.txt");
src.setRemoteDirectory("/sub1");
src.setMaxObjectsPerBatch(15);
src.setTemporarySuffix(".temp");
src.setAcceptSubFolders(true);
src.setDirectory(new LiteralExpression(temp.getRoot().getAbsolutePath()));
src.setBeanFactory(Mockito.mock(BeanFactory.class));
src.afterPropertiesSet();
File file = src.receive().getPayload();
assertEquals(temp.getRoot().getAbsoluteFile() +
File.separator + "sub1" + File.separator + "test.txt", file.getAbsolutePath());
file = src.receive().getPayload();
assertEquals(temp.getRoot().getAbsoluteFile() +
File.separator + "sub1" + File.separator + "sub11" + File.separator + "test.txt", file.getAbsolutePath());
Message<File> message = src.receive();
assertNull(message);
}
}

264
src/test/java/org/springframework/integration/aws/s3/AmazonS3ObjectBuilderTests.java
View File

@@ -1,264 +0,0 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.aws.s3;
import static org.springframework.integration.aws.s3.core.ObjectPermissions.READ;
import static org.springframework.integration.aws.s3.core.ObjectPermissions.READ_ACP;
import static org.springframework.integration.aws.s3.core.ObjectPermissions.WRITE_ACP;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.springframework.integration.aws.s3.core.AmazonS3Object;
import org.springframework.integration.aws.s3.core.AmazonS3ObjectACL;
import org.springframework.integration.aws.s3.core.Grantee;
import org.springframework.integration.aws.s3.core.GranteeType;
import org.springframework.integration.aws.s3.core.ObjectGrant;
import org.springframework.integration.aws.s3.core.ObjectPermissions;
import org.springframework.integration.test.util.TestUtils;
/**
* The test case for the class {@link AmazonS3ObjectBuilder}
*
* @author Amol Nayak
*
* @since 0.5
*
*/
public class AmazonS3ObjectBuilderTests {
@Rule
public TemporaryFolder temp = new TemporaryFolder();
private static final String GROUP_GRANTEE = "http://acs.amazonaws.com/groups/global/AllUsers";
private static final String EMAIL_GRANTEE = "test@test.com";
private static final String CANONICAL_GRANTEE = "12345678900987654321abcdefabcdeab12345678900987654321abcdefabcde";
/**
* Tries to construct the object with a null file instance
*/
@Test(expected=IllegalArgumentException.class)
public void withNullFile() {
AmazonS3ObjectBuilder.getInstance().fromFile(null);
}
/**
* Tries to construct the file from a non existent file
*/
@Test(expected=IllegalArgumentException.class)
public void withNonExistentFile() {
AmazonS3ObjectBuilder.getInstance().fromFile(new File("somejunkfile"));
}
/**
* Tries to construct the file which is a directory
*/
@Test(expected=IllegalArgumentException.class)
public void withADirectory() throws IOException {
File dir = temp.newFolder("tempdir");
AmazonS3ObjectBuilder.getInstance().fromFile(dir);
dir.delete();
}
/**
* Tries to construct the {@link AmazonS3Object} from a null location
*/
@Test(expected=IllegalArgumentException.class)
public void withNullPathString() {
AmazonS3ObjectBuilder.getInstance().fromLocation(null);
}
/**
* Tries to construct the {@link AmazonS3Object} from a valid file
*/
@Test
public void withValidFile() throws Exception {
File file = temp.newFile("temp.txt");
String pathname = file.getAbsolutePath();
AmazonS3ObjectBuilder builder =
AmazonS3ObjectBuilder.getInstance().fromLocation(pathname);
file = TestUtils.getPropertyValue(builder, "file", File.class);
Assert.assertNotNull(file);
Assert.assertEquals(pathname, file.getAbsolutePath());
}
/**
* Tries to construct the {@link AmazonS3Object} from an {@link InputStream}
*/
@Test
public void withInputStream()throws Exception {
File tempFile = temp.newFile("Temp.txt");
InputStream in = new FileInputStream(tempFile);
AmazonS3ObjectBuilder builder =
AmazonS3ObjectBuilder.getInstance().fromInputStream(in);
Assert.assertNotNull(TestUtils.getPropertyValue(builder, "in", InputStream.class));
in.close();
}
/**
* Tries to construct the {@link AmazonS3Object} from an {@link InputStream} and a {@link File}
* instance
*/
@Test(expected=IllegalArgumentException.class)
public void withBothinputStreamAndFile() throws Exception {
File tempFile = temp.newFile("Temp.txt");
FileInputStream in = new FileInputStream(tempFile);
AmazonS3ObjectBuilder
.getInstance()
.fromInputStream(in)
.fromFile(tempFile);
in.close();
}
/**
* Constructs the {@link AmazonS3Object} with some user metadata
*/
@Test
public void withUserMetadata() throws Exception {
File tempFile = temp.newFile("Temp.txt");
AmazonS3ObjectBuilder builder = AmazonS3ObjectBuilder
.getInstance()
.fromFile(tempFile)
.withUserMetaData(Collections.singletonMap("Key", "Value"));
Assert.assertNotNull(TestUtils.getPropertyValue(builder, "userMetaData", Map.class));
}
/**
* Constructs the {@link AmazonS3Object} with some metadata
*/
@Test
public void withMetadata() throws Exception {
File tempFile = temp.newFile("Temp.txt");
AmazonS3ObjectBuilder builder = AmazonS3ObjectBuilder
.getInstance()
.fromFile(tempFile)
.withMetaData(Collections.singletonMap("Key", (Object)"Value"));
Assert.assertNotNull(TestUtils.getPropertyValue(builder, "metaData", Map.class));
}
/**
* Constructs the {@link AmazonS3Object} with an invalid ACL identifier
*/
@Test
public void withValidACL() throws Exception {
File tempFile = temp.newFile("Temp.txt");
Map<String, Collection<String>> acls = generateObjectACLS();
AmazonS3ObjectBuilder builder = AmazonS3ObjectBuilder
.getInstance()
.fromFile(tempFile)
.withObjectACL(acls);
AmazonS3ObjectACL acl = TestUtils.getPropertyValue(builder, "objectACL", AmazonS3ObjectACL.class);
assertGrants(acl);
}
/**
* Builds a complete object with all the possible attributes and checks
* if those are properly populated in the constructed {@link AmazonS3Object}
*/
@Test
public void withCompleteObjectFromFile() throws Exception {
File tempFile = temp.newFile("Temp.txt");
Map<String, Collection<String>> acls = generateObjectACLS();
Map<String, String> userMetadata = Collections.singletonMap("Key", "Value");
Map<String, Object> metadata = Collections.singletonMap("Key", (Object)"Value");
AmazonS3ObjectBuilder builder = AmazonS3ObjectBuilder
.getInstance()
.fromFile(tempFile)
.withObjectACL(acls)
.withMetaData(metadata)
.withUserMetaData(userMetadata);
AmazonS3Object object = builder.build();
assertGrants(object.getObjectACL());
Assert.assertEquals(userMetadata, object.getUserMetaData());
Assert.assertEquals(metadata, object.getMetaData());
Assert.assertEquals(tempFile, object.getFileSource());
Assert.assertNull(object.getInputStream());
}
/**
* Builds an object with an {@link InputStream}
*/
@Test
public void wothObjectFromStream() throws Exception {
File tempFile = temp.newFile("Temp.txt");
InputStream in = new FileInputStream(tempFile);
AmazonS3ObjectBuilder builder = AmazonS3ObjectBuilder
.getInstance()
.fromInputStream(in);
AmazonS3Object object = builder.build();
Assert.assertNull(object.getFileSource());
Assert.assertEquals(in,object.getInputStream());
in.close();
}
/**
* Generate the object test ACLS
*/
private Map<String, Collection<String>> generateObjectACLS() {
Map<String, Collection<String>> acls = new HashMap<String, Collection<String>>();
acls.put(CANONICAL_GRANTEE,
Arrays.asList("write acp"));
acls.put(EMAIL_GRANTEE,
Arrays.asList("read acp", "write acp"));
acls.put(GROUP_GRANTEE,
Arrays.asList("read"));
return acls;
}
/**
* Checks for the grants in the object
*
*/
private void assertGrants(AmazonS3ObjectACL acl) {
Set<ObjectGrant> grants = acl.getGrants();
Assert.assertEquals(4, grants.size());
for(ObjectGrant grant:grants) {
Grantee grantee = grant.getGrantee();
GranteeType type = grantee.getGranteeType();
ObjectPermissions permission = grant.getPermission();
if(type == GranteeType.CANONICAL_GRANTEE_TYPE) {
Assert.assertEquals(CANONICAL_GRANTEE, grantee.getIdentifier());
Assert.assertEquals(WRITE_ACP,permission);
}
else if(type == GranteeType.EMAIL_GRANTEE_TYPE){
Assert.assertEquals(EMAIL_GRANTEE, grantee.getIdentifier());
Assert.assertTrue(permission == WRITE_ACP || permission == READ_ACP);
}
else if(type == GranteeType.GROUP_GRANTEE_TYPE) {
Assert.assertEquals(GROUP_GRANTEE, grantee.getIdentifier());
Assert.assertEquals(READ,permission);
}
}
}
}

189
src/test/java/org/springframework/integration/aws/s3/AmazonS3OperationsMockingUtil.java
View File

@@ -1,189 +0,0 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.aws.s3;
import static org.mockito.Matchers.anyInt;
import static org.mockito.Matchers.anyString;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import static org.springframework.integration.aws.s3.InboundFileSynchronizationImpl.CONTENT_MD5;
import java.io.ByteArrayInputStream;
import java.util.ArrayList;
import java.util.Collections;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.codec.binary.Hex;
import org.mockito.invocation.InvocationOnMock;
import org.mockito.stubbing.Answer;
import org.springframework.integration.aws.s3.core.AmazonS3Object;
import org.springframework.integration.aws.s3.core.AmazonS3Operations;
import org.springframework.integration.aws.s3.core.PaginatedObjectsView;
import org.springframework.integration.aws.s3.core.S3ObjectSummary;
import org.springframework.util.StringUtils;
/**
* The utility class for mocking the {@link AmazonS3Operations}
*
* @author Amol Nayak
*
* @since 0.5
*/
public final class AmazonS3OperationsMockingUtil {
public static final String BUCKET = "com.si.aws.test.bucket";
private static final List<S3ObjectSummary> summary = new ArrayList<S3ObjectSummary>();
private static final Map<String, String[]> objectDetails = new HashMap<String, String[]>();
private AmazonS3OperationsMockingUtil() {
throw new AssertionError("Cannot instantiate utility class");
}
public static final AmazonS3Operations mockS3Operations() {
AmazonS3Operations operations;
PaginatedObjectsView view = new PaginatedObjectsView() {
@Override
public boolean hasMoreResults() {
return false;
}
@Override
public List<S3ObjectSummary> getObjectSummary() {
return summary;
}
@Override
public String getNextMarker() {
return null;
}
};
operations = mock(AmazonS3Operations.class);
when(operations.listObjects(anyString(), anyString(), anyString(), anyInt()))
.thenReturn(view);
when(operations.getObject(anyString(), anyString(), anyString()))
.then(new Answer<AmazonS3Object>() {
public AmazonS3Object answer(InvocationOnMock invocation)
throws Throwable {
String folderName = (String)invocation.getArguments()[1];
String fileName = (String)invocation.getArguments()[2];
if(StringUtils.hasText(folderName)) {
if(folderName.startsWith("/")) {
folderName = folderName.substring(1);
}
if(folderName.endsWith("/")) {
folderName = folderName + "/";
}
}
else {
folderName = "";
}
String[] object = objectDetails.get(folderName + fileName);
AmazonS3Object s3Object;
if(object != null) {
s3Object = new AmazonS3ObjectBuilder()
.fromInputStream(new ByteArrayInputStream(object[1].getBytes()))
.withUserMetaData(Collections.singletonMap(CONTENT_MD5, object[2]))
.build();
}
else {
s3Object = null;
}
return s3Object;
}
});
return operations;
}
/**
* The private helper method that is used to mock the {@link AmazonS3Operations#listObjects(String, String, String, int)
* method
*
* The method accepts a List of object[] where each element of object array has the
* following significance
*
* object[0] is the key of the file
* object[1] is the content of the file
* object[2] is the MD5 content to be used (optional)
* object[3] is the etag of the object
*
* @param listObjects
*/
public static void mockAmazonS3Operations(final List<String[]> listObjects) {
summary.clear();
for(String[] listObject:listObjects) {
addToSummaryList(listObject);
objectDetails.put(listObject[0], listObject);
}
}
/**
* Creates a {@link S3ObjectSummary} with the given details and adds it to the summary object list
* @param listObject
*/
private static void addToSummaryList(final String[] listObject) {
summary.add(
new S3ObjectSummary() {
@Override
public long getSize() {
return 0;
}
@Override
public Date getLastModified() {
return null;
}
@Override
public String getKey() {
return listObject[0];
}
@Override
public String getETag() {
if(listObject[3] != null) {
return listObject[3];
}
else {
byte[] b64 = Base64.decodeBase64((listObject[2]).getBytes());
return Hex.encodeHexString(b64);
}
}
@Override
public String getBucketName() {
return null;
}
}
);
}
}

126
src/test/java/org/springframework/integration/aws/s3/DefaultFileNameGenerationStrategyTests.java
View File

@@ -1,126 +0,0 @@
/*
* Copyright 2002-2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.aws.s3;
import java.io.File;
import java.io.IOException;
import java.util.UUID;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.mockito.Mockito;
import org.springframework.beans.factory.BeanFactory;
import org.springframework.integration.support.MessageBuilder;
import org.springframework.messaging.Message;
/**
* The test class for {@link DefaultFileNameGenerationStrategy}
*
* @author Amol Nayak
* @author Rob Harrop
* @author Artem Bilan
*
* @since 0.5
*
*/
public class DefaultFileNameGenerationStrategyTests {
@Rule
public TemporaryFolder tempFolder = new TemporaryFolder();
/**
* Tests with the file name present in the predetermined header "file_name" of the message
*/
@Test
public void withNameInHeader() {
Message<String> message = MessageBuilder.withPayload("SomeString")
.setHeader(AmazonS3MessageHeaders.FILE_NAME, "FileName.txt")
.build();
DefaultFileNameGenerationStrategy strategy = new DefaultFileNameGenerationStrategy();
strategy.setBeanFactory(Mockito.mock(BeanFactory.class));
Assert.assertEquals("FileName.txt", strategy.generateFileName(message));
}
/**
* Tests with a payload as a temp file payload
* @throws IOException
*/
@Test
public void withATempFile() throws IOException {
final File file = tempFolder.newFile("TempFile.txt.writing");
Message<File> message = MessageBuilder.withPayload(file)
.build();
DefaultFileNameGenerationStrategy strategy = new DefaultFileNameGenerationStrategy();
strategy.setBeanFactory(Mockito.mock(BeanFactory.class));
Assert.assertEquals("TempFile.txt", strategy.generateFileName(message));
file.delete();
}
/**
* Tests with a payload as a temp file payload
* @throws IOException
*/
@Test
public void withANonTempFile() throws IOException {
final File file = tempFolder.newFile("TempFile.txt");
Message<File> message = MessageBuilder.withPayload(file)
.build();
DefaultFileNameGenerationStrategy strategy = new DefaultFileNameGenerationStrategy();
strategy.setBeanFactory(Mockito.mock(BeanFactory.class));
Assert.assertEquals("TempFile.txt", strategy.generateFileName(message));
file.delete();
}
/**
* Tests with a payload as a temp file payload
*/
@Test
public void withMessageIdName() {
Message<String> message = MessageBuilder.withPayload("String")
.build();
DefaultFileNameGenerationStrategy strategy = new DefaultFileNameGenerationStrategy();
strategy.setBeanFactory(Mockito.mock(BeanFactory.class));
UUID uid = message.getHeaders().getId();
Assert.assertEquals(uid.toString() + ".ext", strategy.generateFileName(message));
}
/**
* Tests with the file name generation expression as a null value
*/
@Test(expected=IllegalArgumentException.class)
public void withNullExprssion() {
DefaultFileNameGenerationStrategy strategy = new DefaultFileNameGenerationStrategy();
strategy.setBeanFactory(Mockito.mock(BeanFactory.class));
strategy.setFileNameExpression(null);
}
/**
* Tests with a null value for temporary suffix
*/
@Test(expected=IllegalArgumentException.class)
public void withNullTemporarySuffix() {
DefaultFileNameGenerationStrategy strategy = new DefaultFileNameGenerationStrategy();
strategy.setBeanFactory(Mockito.mock(BeanFactory.class));
strategy.setTemporarySuffix(null);
}
}

125
src/test/java/org/springframework/integration/aws/s3/FileNameFilterTests.java
View File

@@ -1,125 +0,0 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.aws.s3;
import org.junit.Assert;
import org.junit.Test;
/**
*
* The test cases for various {@link FileNameFilter} implementations.
*
* @author Amol Nayak
*
* @since 0.5
*
*/
public class FileNameFilterTests {
/**
* The case sets the folder of the filter to null so that it accepts all the files in the same
* folder, the {@link AbstractFileNameFilter#isAcceptSubFolders() returns false.
*
*/
@Test
public void acceptAllFilesWithoutSubfolders() {
AbstractFileNameFilter filter = new AlwaysTrueFileNamefilter();
filter.setAcceptSubFolders(false);
Assert.assertTrue(filter.accept("SomeFile.txt"));
Assert.assertFalse(filter.accept("somessubfolder/SomeFile.txt"));
}
/**
* The case sets the folder of the filter to null so that it accepts all the files in the same
* folder and the sub folders, the {@link AbstractFileNameFilter#isAcceptSubFolders() returns true.
*
*/
@Test
public void acceptAllFilesWithSubfolders() {
AbstractFileNameFilter filter = new AlwaysTrueFileNamefilter();
filter.setAcceptSubFolders(true);
Assert.assertTrue(filter.accept("SomeFile.txt"));
Assert.assertTrue(filter.accept("somessubfolder/SomeFile.txt"));
}
/**
* The test case sets a sub folder for the search and sets the
* {@link AbstractFileNameFilter#setAcceptSubFolders(boolean) to false
*/
@Test
public void acceptInSubfolderWithoutSubfolder() {
AbstractFileNameFilter filter = new AlwaysTrueFileNamefilter();
filter.setFolderName("/subfolder");
filter.setAcceptSubFolders(false);
Assert.assertFalse(filter.accept("FileName.txt"));
Assert.assertTrue(filter.accept("subfolder/FileName.txt"));
Assert.assertFalse(filter.accept("subfolder/anothersf/FileName.txt"));
}
/**
* The test case sets a sub folder for the search and sets the
* {@link AbstractFileNameFilter#setAcceptSubFolders(boolean) to true
*/
@Test
public void acceptInSubfolderWithSubfolder() {
AbstractFileNameFilter filter = new AlwaysTrueFileNamefilter();
filter.setFolderName("/subfolder");
filter.setAcceptSubFolders(true);
Assert.assertFalse(filter.accept("FileName.txt"));
Assert.assertTrue(filter.accept("subfolder/FileName.txt"));
Assert.assertTrue(filter.accept("subfolder/anothersf/FileName.txt"));
}
/**
* Tests the regex file filter
*/
@Test
public void regexTest() {
//accept only file names with name in lower case and ends with .txt
AbstractFileNameFilter filter = new RegexFileNameFilter("[a-z]+\\.txt");
filter.setAcceptSubFolders(true);
Assert.assertTrue(filter.accept("test.txt"));
Assert.assertFalse(filter.accept("Test.txt"));
Assert.assertFalse(filter.accept("test123.txt"));
Assert.assertFalse(filter.accept("test.tx"));
Assert.assertFalse(filter.accept("test"));
Assert.assertTrue(filter.accept("test/test.txt"));
Assert.assertTrue(filter.accept("test/Test/12/test.txt"));
Assert.assertFalse(filter.accept("test/Test/12/test.tx"));
Assert.assertFalse(filter.accept("test/Test/12/Test.txt"));
}
/**
* Tests the wildcard file filter
*/
@Test
public void wildCardTest() {
//accept only file names with name in lower case and ends with .txt
AbstractFileNameFilter filter = new WildcardFileNameFilter("*.txt");
filter.setAcceptSubFolders(true);
Assert.assertTrue(filter.accept("test.txt"));
Assert.assertTrue(filter.accept("Test.txt"));
Assert.assertTrue(filter.accept("test123.txt"));
Assert.assertFalse(filter.accept("test.tx"));
Assert.assertFalse(filter.accept("test"));
Assert.assertTrue(filter.accept("test/test.txt"));
Assert.assertTrue(filter.accept("test/Test/12/test.txt"));
Assert.assertTrue(filter.accept("test/Test/12/Test.txt"));
Assert.assertFalse(filter.accept("test/Test/12/Test.ext"));
}
}

462
src/test/java/org/springframework/integration/aws/s3/InboundFileSynchronizationImplTests.java
View File

@@ -1,462 +0,0 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.aws.s3;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.springframework.integration.aws.common.AWSTestUtils.assertFileContent;
import static org.springframework.integration.aws.common.AWSTestUtils.getContentsRecursively;
import static org.springframework.integration.aws.common.AWSTestUtils.md5Hash;
import static org.springframework.integration.aws.common.AWSTestUtils.writeToFile;
import static org.springframework.integration.aws.s3.AmazonS3OperationsMockingUtil.BUCKET;
import static org.springframework.integration.aws.s3.AmazonS3OperationsMockingUtil.mockAmazonS3Operations;
import static org.springframework.integration.aws.s3.AmazonS3OperationsMockingUtil.mockS3Operations;
import java.io.File;
import java.util.Arrays;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.codec.binary.Hex;
import org.junit.BeforeClass;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.springframework.integration.aws.s3.core.AmazonS3Operations;
import org.springframework.integration.test.util.TestUtils;
/**
* Test class for {@link InboundFileSynchronizationImpl}
*
* @author Amol Nayak
*
* @since 0.5
*
*/
public class InboundFileSynchronizationImplTests {
private static AmazonS3Operations operations;
@Rule
public final TemporaryFolder tempFolder = new TemporaryFolder();
@BeforeClass
public static void setup() {
operations = mockS3Operations();
}
private InboundFileSynchronizationImpl getInboundFileSynchronizationImpl() {
//uncomment the below code if you want to execute against the actual bucket
//but before that you need to do the following
//create a bucket and set that in the constant BUCKET in the class
//1. add a file test.txt with content test.txt to the root.
//2. create a folder sub1 and add a file test.txt to it with content sub1/test.txt.
//3. create a folder sub1/sub11 and add a file test.txt to it with content sub1/sub11/test.txt.
//4. create a folder sub2 and add a file test.txt to it with content sub/test.txt.
// DefaultAmazonS3Operations operations = new DefaultAmazonS3Operations(AmazonWSTestUtils.getCredentials());
// try {
// operations.afterPropertiesSet();
// } catch (Exception e) {
// e.printStackTrace();
// }
InboundFileSynchronizationImpl sync = new InboundFileSynchronizationImpl(operations,
new InboundLocalFileOperationsImpl());
return sync;
}
/**
* Tests with {@link AmazonS3Operations} instance as null
*/
@Test(expected=IllegalArgumentException.class)
public void withS3OperationsAsNull() {
new InboundFileSynchronizationImpl(null, new InboundLocalFileOperationsImpl());
}
/**
* Tests with {@link InboundLocalFileOperations} instance as null
*/
@Test(expected=IllegalArgumentException.class)
public void withLocalFileOperationsAsNull() {
new InboundFileSynchronizationImpl(operations, null);
}
/**
* Tests after setting both the wildcard and filename regex
*/
@Test(expected=IllegalArgumentException.class)
public void withBothWildCardAndRegex() throws Exception {
InboundFileSynchronizationImpl sync = getInboundFileSynchronizationImpl();
sync.setFileWildcard("*.txt");
sync.setFileNamePattern("[a-z]+\\.txt");
sync.afterPropertiesSet();
}
/**
* Sets non of regex and wildcard
*/
@Test
public void withNoneOfRegexAndWildcard() throws Exception {
InboundFileSynchronizationImpl impl = getInboundFileSynchronizationImpl();
impl.afterPropertiesSet();
assertEquals(AlwaysTrueFileNamefilter.class,
TestUtils.getPropertyValue(impl, "filter", FileNameFilter.class).getClass());
}
/**
* Tests after setting filename regex only
*/
@Test
public void withRegexOnly() throws Exception {
InboundFileSynchronizationImpl sync = getInboundFileSynchronizationImpl();
sync.setFileNamePattern("[a-z]+\\.txt");
sync.afterPropertiesSet();
FileNameFilter filter = TestUtils.getPropertyValue(sync, "filter",FileNameFilter.class);
assertNotNull(filter);
assertEquals(RegexFileNameFilter.class, filter.getClass());
assertEquals("[a-z]+\\.txt",
TestUtils.getPropertyValue(filter, "filter.pattern.pattern", String.class));
}
/**
* Tests after setting filename wildcard only
*/
@Test
public void withWildcardOnly() throws Exception {
InboundFileSynchronizationImpl sync = getInboundFileSynchronizationImpl();
sync.setFileWildcard("*.txt");
sync.afterPropertiesSet();
FileNameFilter filter = TestUtils.getPropertyValue(sync, "filter",FileNameFilter.class);
assertNotNull(filter);
assertEquals(WildcardFileNameFilter.class, filter.getClass());
assertEquals("*.txt",
TestUtils.getPropertyValue(filter, "filter.wildcards", String[].class)[0]);
}
/**
* Sets the {@link InboundFileSynchronizationImpl#setAcceptSubFolders(boolean) as true
*/
@Test
public void withAcceptSubfolderAsTrue() throws Exception {
InboundFileSynchronizationImpl impl = getInboundFileSynchronizationImpl();
impl.setAcceptSubFolders(true);
impl.afterPropertiesSet();
assertTrue(TestUtils.getPropertyValue(impl, "filter.acceptSubFolders",Boolean.class).booleanValue());
assertTrue(TestUtils.getPropertyValue(impl, "fileOperations.createDirectoriesIfRequired",
Boolean.class).booleanValue());
}
/**
* Invokes with remote directory as / and create directory set to true
*/
@Test
public void withRemoteAsRootAndCreateDirectoryToTrue() throws Exception {
setupMock();
String rootDirectoryPath = tempFolder.getRoot().getAbsolutePath();
String path = String.format("%s%s%s",
rootDirectoryPath,File.separator,"test.txt");
File fileOne = new File(path);
path = String.format("%s%s%s%s%s",
rootDirectoryPath,File.separator,"sub1",File.separator,"test.txt");
File fileTwo = new File(path);
path = String.format("%s%s%s%s%s%s%s",
rootDirectoryPath,File.separator,"sub1",File.separator,"sub11",File.separator,"test.txt");
File fileThree = new File(path);;
path = String.format("%s%s%s%s%s",
rootDirectoryPath,File.separator,"sub2",File.separator,"test.txt");
File fileFour = new File(path);
assertFalse(fileOne.exists());
assertFalse(fileTwo.exists());
assertFalse(fileThree.exists());
assertFalse(fileFour.exists());
InboundFileSynchronizationImpl impl = getInboundFileSynchronizationImpl();
impl.setAcceptSubFolders(true);
impl.afterPropertiesSet();
impl.synchronizeToLocalDirectory(tempFolder.getRoot(), BUCKET, "/");
assertTrue(fileOne.exists());
assertFileContent(fileOne, "test.txt");
assertTrue(fileTwo.exists());
assertFileContent(fileTwo, "sub1/test.txt");
assertTrue(fileThree.exists());
assertFileContent(fileThree, "sub1/sub11/test.txt");
assertTrue(fileFour.exists());
assertFileContent(fileFour, "sub2/test.txt");
assertEquals(4, getContentsRecursively(tempFolder.getRoot()).size());
}
/**
* Invokes with remote directory as / and create directory set to false
*/
@Test
public void withRemoteAsRootAndCreateDirectoryToFalse() throws Exception {
setupMock();
String rootDirectoryPath = tempFolder.getRoot().getAbsolutePath();
String path = String.format("%s%s%s",
rootDirectoryPath,File.separator,"test.txt");
File fileOne = new File(path);
assertFalse(fileOne.exists());
InboundFileSynchronizationImpl impl = getInboundFileSynchronizationImpl();
impl.setAcceptSubFolders(false);
impl.afterPropertiesSet();
impl.synchronizeToLocalDirectory(tempFolder.getRoot(), BUCKET, "/");
assertTrue(fileOne.exists());
assertFileContent(fileOne, "test.txt");
assertEquals(1, getContentsRecursively(tempFolder.getRoot()).size());
}
/**
* Invokes with remote directory as /sub1 and create directory set to true
*/
@Test
public void withRemoteAssub1AndCreateDirectoryToTrue() throws Exception {
setupMock();
String rootDirectoryPath = tempFolder.getRoot().getAbsolutePath();
String path = String.format("%s%s%s%s%s",
rootDirectoryPath,File.separator,"sub1",File.separator,"test.txt");
File fileOne = new File(path);
path = String.format("%s%s%s%s%s%s%s",
rootDirectoryPath,File.separator,"sub1",File.separator,"sub11",File.separator,"test.txt");
File fileTwo = new File(path);
assertFalse(fileOne.exists());
assertFalse(fileTwo.exists());
InboundFileSynchronizationImpl impl = getInboundFileSynchronizationImpl();
impl.setAcceptSubFolders(true);
impl.afterPropertiesSet();
impl.synchronizeToLocalDirectory(tempFolder.getRoot(), BUCKET, "/sub1");
assertTrue(fileOne.exists());
assertFileContent(fileOne, "sub1/test.txt");
assertTrue(fileTwo.exists());
assertFileContent(fileTwo, "sub1/sub11/test.txt");
assertEquals(2, getContentsRecursively(tempFolder.getRoot()).size());
}
/**
* Invokes with remote directory as /sub1 and create directory set to false
*/
@Test
public void withRemoteAssub1AndCreateDirectoryToFalse() throws Exception {
setupMock();
String rootDirectoryPath = tempFolder.getRoot().getAbsolutePath();
String path = String.format("%s%s%s%s%s",
rootDirectoryPath,File.separator,"sub1",File.separator,"test.txt");
File file = new File(path);
assertFalse(file.exists());
InboundFileSynchronizationImpl impl = getInboundFileSynchronizationImpl();
impl.setAcceptSubFolders(false);
impl.afterPropertiesSet();
impl.synchronizeToLocalDirectory(tempFolder.getRoot(), BUCKET, "/sub1");
assertTrue(file.exists());
assertFileContent(file, "sub1/test.txt");
assertEquals(1, getContentsRecursively(tempFolder.getRoot()).size());
}
/**
* Invokes with remote directory as / and create directory set to false
* The two files test.txt and sub1/test.txt would already be present on
* the file system. Both test.txt and sub/test.txt will have content different that the remote one.
* test.txt will be replaced and sub/test.txt will not be replaced.
*
*/
@Test
public void withRemoteAsRootAndCreateDirectoryToFalse2() throws Exception {
setupMock();
String rootDirectoryPath = tempFolder.getRoot().getAbsolutePath();
//create test.txt and sub1/test.txt
String path = String.format("%s%s%s",
rootDirectoryPath,File.separator,"test.txt");
writeToFile(path, "OldContents");
File fileOne = new File(path);
assertTrue(fileOne.exists());
assertFileContent(fileOne, "OldContents");
path = String.format("%s%s%s%s%s",
rootDirectoryPath,File.separator,"sub1",File.separator,"test.txt");
writeToFile(path, "OldContents");
File fileTwo = new File(path);
assertTrue(fileTwo.exists());
assertFileContent(fileTwo, "OldContents");
InboundFileSynchronizationImpl impl = getInboundFileSynchronizationImpl();
impl.setAcceptSubFolders(false);
impl.afterPropertiesSet();
impl.synchronizeToLocalDirectory(tempFolder.getRoot(), BUCKET, "/");
assertTrue(fileOne.exists());
assertFileContent(fileOne, "test.txt");
assertTrue(fileTwo.exists());
assertFileContent(fileTwo, "OldContents");
assertEquals(2, getContentsRecursively(tempFolder.getRoot()).size());
}
/**
* Invokes with remote directory as / and create directory set to true
* The two files test.txt and sub1/test.txt would already be present on
* the file system. Both test.txt and sub/test.txt will have content different that the remote one.
* test.txt and sub/test.txt both will be replaced.
*
*/
@Test
public void withRemoteAsRootAndCreateDirectoryToTrue2() throws Exception {
setupMock();
String rootDirectoryPath = tempFolder.getRoot().getAbsolutePath();
//create test.txt and sub1/test.txt
String path = String.format("%s%s%s",
rootDirectoryPath,File.separator,"test.txt");
writeToFile(path, "OldContents");
File fileOne = new File(path);
assertTrue(fileOne.exists());
assertFileContent(fileOne, "OldContents");
path = String.format("%s%s%s%s%s",
rootDirectoryPath,File.separator,"sub1",File.separator,"test.txt");
writeToFile(path, "OldContents");
File fileTwo = new File(path);
assertTrue(fileTwo.exists());
assertFileContent(fileTwo, "OldContents");
InboundFileSynchronizationImpl impl = getInboundFileSynchronizationImpl();
impl.setAcceptSubFolders(true);
impl.afterPropertiesSet();
impl.synchronizeToLocalDirectory(tempFolder.getRoot(), BUCKET, "/");
assertTrue(fileOne.exists());
assertFileContent(fileOne, "test.txt");
assertTrue(fileTwo.exists());
assertFileContent(fileTwo, "sub1/test.txt");
assertEquals(4, getContentsRecursively(tempFolder.getRoot()).size());
}
/**
* The case is slightly different then previous ones.
* We list from the /sub1 with /sub1/test.txt and /sub1/sub11/test.txt
* having different contents, however the etag of
* /sub1/sub11/test.txt is same as remote one hence the local one should not get replaced
* where as /sub1/test.txt should.
*
*/
@Test
public void withRemoteAsRootAndCreateDirectoryToTrue3() throws Exception {
mockAmazonS3Operations(Arrays.asList(
new String[]{"test.txt","test.txt",md5Hash("test.txt"),null},
new String[]{"sub1/test.txt","sub1/test.txt",md5Hash("sub1/test.txt"),null},
new String[]{"sub1/sub11/test.txt","sub1/sub11/test.txt",md5Hash("OldContents"),null},
new String[]{"sub2/test.txt","sub2/test.txt",md5Hash("sub2/test.txt"),null}
));
withinSub1FolderTests(true);
}
/**
* The scenario tests by listing the directory /sub1 which has two files
* /sub1/test.txt and /sub1/sub11/test.txt. The MD5 of the file will be absent and the etag is
* for MultiUpload. This should force replace the file irrespective of the content.
*/
@Test
public void withMultipartUploadForceReplace() throws Exception {
mockAmazonS3Operations(Arrays.asList(
new String[]{"sub1/test.txt","sub1/test.txt",null,
new String(Hex.encodeHex(Base64.decodeBase64(md5Hash("SomeContentSub1").getBytes()))) + "-1"},
new String[]{"sub1/sub11/test.txt","sub1/sub11/test.txt",null,
new String(Hex.encodeHex(Base64.decodeBase64(md5Hash("SomeContentSub1/Sub11").getBytes()))) + "-1"}
));
withinSub1FolderTests(false);
}
/**
* The scenario will test with two files present in /sub1 directory, /sub1/test.txt and
* /sub1/sub11/test.txt. Now both these files have multipart upload etag but both have
* MD5 hash in the user's metadata. The contents of both the files is different than the one
* on remote but /sub/test.txt has MD5 sum same as remote, so this should not get replaced
*
* @throws Exception
*/
@Test
public void withMultipartUploadWithMD5Metadata() throws Exception {
mockAmazonS3Operations(Arrays.asList(
new String[]{"sub1/test.txt","sub1/test.txt",md5Hash("sub1/test.txt"),
new String(Hex.encodeHex(Base64.decodeBase64(md5Hash("sub1/test.txt").getBytes()))) + "-1"},
new String[]{"sub1/sub11/test.txt","sub1/sub11/test.txt",md5Hash("OldContents"),
new String(Hex.encodeHex(Base64.decodeBase64(md5Hash("OldContents").getBytes()))) + "-1"}
));
withinSub1FolderTests(true);
}
/**
* Private method that extracts the common assertion logic for the files in sub1 folder
* @throws Exception
*/
private void withinSub1FolderTests(boolean acceptSubfolder) throws Exception {
String rootDirectoryPath = tempFolder.getRoot().getAbsolutePath();
//create sub1/sub11/test.txt and sub1/test.txt
String path = String.format("%s%s%s%s%s%s%s",
rootDirectoryPath,File.separator,"sub1",File.separator,"sub11",File.separator,"test.txt");
writeToFile(path, "OldContents");
File fileOne = new File(path);
assertTrue(fileOne.exists());
assertFileContent(fileOne, "OldContents");
path = String.format("%s%s%s%s%s",
rootDirectoryPath,File.separator,"sub1",File.separator,"test.txt");
writeToFile(path, "OldContents");
File fileTwo = new File(path);
assertTrue(fileTwo.exists());
assertFileContent(fileTwo, "OldContents");
InboundFileSynchronizationImpl impl = getInboundFileSynchronizationImpl();
impl.setAcceptSubFolders(acceptSubfolder);
impl.afterPropertiesSet();
impl.synchronizeToLocalDirectory(tempFolder.getRoot(), BUCKET, "/sub1");
assertTrue(fileOne.exists());
assertFileContent(fileOne, "OldContents");
assertTrue(fileTwo.exists());
assertFileContent(fileTwo, "sub1/test.txt");
assertEquals(2, getContentsRecursively(tempFolder.getRoot()).size());
}
/**
* Private helper method that will be setup mock s3 operations to give an illusion
* that it has 4 objects in the remote bucket
*
*/
private void setupMock() {
mockAmazonS3Operations(Arrays.asList(
new String[]{"test.txt","test.txt",md5Hash("test.txt"),null},
new String[]{"sub1/test.txt","sub1/test.txt",md5Hash("sub1/test.txt"),null},
new String[]{"sub1/sub11/test.txt","sub1/sub11/test.txt",md5Hash("sub1/sub11/test.txt"),null},
new String[]{"sub2/test.txt","sub2/test.txt",md5Hash("sub2/test.txt"),null}
));
}
}

177
src/test/java/org/springframework/integration/aws/s3/InboundLocalFileOperationsImplTests.java
View File

@@ -1,177 +0,0 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.aws.s3;
import static org.springframework.integration.aws.common.AWSTestUtils.assertFileContent;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.util.Collections;
import org.junit.Assert;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.springframework.integration.test.util.TestUtils;
/**
* The test class for {@link InboundLocalFileOperationsImpl} that is used to perform
* operations on local file system
*
* @author Amol Nayak
*
* @since 0.5
*
*/
public class InboundLocalFileOperationsImplTests {
@Rule
public TemporaryFolder tempFolder = new TemporaryFolder();
/**
* Tries registering a null listener with the class
*/
@Test(expected=IllegalArgumentException.class)
public void withNullListener() {
InboundLocalFileOperations operations = new InboundLocalFileOperationsImpl();
operations.addEventListener(null);
}
/**
* Tries setting the listeners which is an empty list
*/
@SuppressWarnings("unchecked")
@Test(expected=IllegalArgumentException.class)
public void setEmptyListeners() {
InboundLocalFileOperations operations = new InboundLocalFileOperationsImpl();
operations.setEventListeners(Collections.EMPTY_LIST);
}
/**
* Test case for setting a temporary suffix that begins with a .
*/
@Test
public void setTempSuffixBeginningWithDot() {
InboundLocalFileOperations operations = new InboundLocalFileOperationsImpl();
operations.setTemporaryFileSuffix(".write");
Assert.assertEquals(".write", TestUtils.getPropertyValue(operations, "tempFileSuffix"));
}
/**
* Test case for setting a temporary suffix that does not begins with a .
*/
@Test
public void setTempSuffixNotBeginningWithDot() {
InboundLocalFileOperations operations = new InboundLocalFileOperationsImpl();
operations.setTemporaryFileSuffix("write");
Assert.assertEquals(".write", TestUtils.getPropertyValue(operations, "tempFileSuffix"));
}
/**
*Since the provided directory is null, we expect an {@link IllegalArgumentException}
*/
@Test(expected=IllegalArgumentException.class)
public void writeWithNullDirectory() throws Exception {
InboundLocalFileOperations operations = new InboundLocalFileOperationsImpl();
operations.writeToFile(null, null, null);
}
/**
*Since the provided file name is null, we expect an {@link IllegalArgumentException}
*/
@Test(expected=IllegalArgumentException.class)
public void writeWithNullFileName() throws Exception {
InboundLocalFileOperations operations = new InboundLocalFileOperationsImpl();
operations.writeToFile(tempFolder.newFolder("Test"), null, null);
}
/**
*Since the provided stream as null, we expect an {@link IllegalArgumentException}
*/
@Test(expected=IllegalArgumentException.class)
public void writeWithNullStream() throws Exception {
InboundLocalFileOperations operations = new InboundLocalFileOperationsImpl();
operations.writeToFile(tempFolder.newFolder("Test"), "TestFile.txt", null);
}
/**
*Provided {@link File} for directory exists and is not a directory
*/
@Test(expected=IllegalArgumentException.class)
public void writeWithExistantNonDirectory() throws Exception {
InboundLocalFileOperations operations = new InboundLocalFileOperationsImpl();
operations.writeToFile(tempFolder.newFile("Test"), "TestFile.txt", new ByteArrayInputStream(new byte[]{}));
}
/**
*Provided {@link File} for directory does not exist exists and the create flag is false
*/
@Test(expected=IllegalArgumentException.class)
public void writeWithNonExistentDirectory() throws Exception {
InboundLocalFileOperations operations = new InboundLocalFileOperationsImpl();
operations.writeToFile(new File(tempFolder.getRoot() + "SomeDir"), "TestFile.txt", new ByteArrayInputStream(new byte[]{}));
}
/**
* Writes some test content to the file
*/
@Test
public void writeTestContentToFile() throws Exception {
InboundLocalFileOperations operations = new InboundLocalFileOperationsImpl();
operations.setCreateDirectoriesIfRequired(true);
File directory = new File(tempFolder.getRoot() + File.separator + "someNestedDir");
File tempFile = new File(directory.getAbsolutePath() + File.separator + "SomeFileName.txt.writing");
File permFile = new File(directory.getAbsolutePath() + File.separator + "SomeFileName.txt");
Assert.assertFalse(tempFile.exists());
Assert.assertFalse(permFile.exists());
operations.writeToFile(directory, "SomeFileName.txt", new ByteArrayInputStream("Some Test Content".getBytes()));
Assert.assertFalse(tempFile.exists());
Assert.assertTrue(permFile.exists());
//Check the content
assertFileContent(permFile, "Some Test Content");
//TODO: Test FileEventHandlers
}
/**
* Writes some test content to the file with teh given target file existent
*/
@Test
public void writeTestContentWithTargetExistent() throws Exception {
InboundLocalFileOperations operations = new InboundLocalFileOperationsImpl();
operations.setCreateDirectoriesIfRequired(true);
File directory = tempFolder.newFolder("someNestedDir");
File tempFile = new File(directory.getAbsolutePath() + File.separator + "SomeFileName.txt.writing");
File permFile = new File(directory.getAbsolutePath() + File.separator + "SomeFileName.txt");
permFile.createNewFile();
//Write Some content
FileOutputStream fos = new FileOutputStream(permFile);
fos.write("Some Old Contents".getBytes());
fos.close();
assertFileContent(permFile, "Some Old Contents");
Assert.assertFalse(tempFile.exists());
Assert.assertTrue(permFile.exists());
operations.writeToFile(directory, "SomeFileName.txt", new ByteArrayInputStream("Some Test Content".getBytes()));
Assert.assertFalse(tempFile.exists());
Assert.assertTrue(permFile.exists());
//Check the content
assertFileContent(permFile, "Some Test Content");
}
}

55
src/test/java/org/springframework/integration/aws/s3/config/xml/AmazonS3DummyOperations.java
View File

@@ -1,55 +0,0 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.aws.s3.config.xml;
import org.springframework.integration.aws.s3.core.AmazonS3Object;
import org.springframework.integration.aws.s3.core.AmazonS3Operations;
import org.springframework.integration.aws.s3.core.PaginatedObjectsView;
/**
* The dummy {@link AmazonS3Operations} for tests
*
* @author Amol Nayak
*
* @since 0.5
*
*/
public class AmazonS3DummyOperations implements AmazonS3Operations {
@Override
public PaginatedObjectsView listObjects(String bucketName,
String folder, String nextMarker, int pageSize) {
return null;
}
@Override
public void putObject(String bucketName, String folder,
String objectName, AmazonS3Object s3Object) {
}
@Override
public AmazonS3Object getObject(String bucketName, String folder,
String objectName) {
return null;
}
@Override
public boolean removeObject(String bucketName, String folder,
String objectName) {
return false;
}
}

86
src/test/java/org/springframework/integration/aws/s3/config/xml/AmazonS3InboundChannelAdapterParserTests.java
View File

@@ -1,86 +0,0 @@
/*
* Copyright 2002-2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.aws.s3.config.xml;
import static org.junit.Assert.assertEquals;
import static org.springframework.integration.test.util.TestUtils.getPropertyValue;
import java.io.File;
import java.net.URI;
import org.junit.Test;
import org.springframework.beans.factory.BeanDefinitionStoreException;
import org.springframework.context.support.ClassPathXmlApplicationContext;
import org.springframework.integration.aws.s3.AmazonS3InboundSynchronizationMessageSource;
import org.springframework.integration.aws.s3.core.AmazonS3Operations;
import org.springframework.integration.aws.s3.core.DefaultAmazonS3Operations;
import org.springframework.integration.endpoint.SourcePollingChannelAdapter;
/**
* The test case class for S3 inbound channel adapter
*
* @author Amol Nayak
* @author Li Wang
* @since 0.5
*
*/
public class AmazonS3InboundChannelAdapterParserTests {
/**
* Tests the inbound channel adapter definition with a valid combination of attributes
*/
@Test
public void withValidAttributeValues() {
ClassPathXmlApplicationContext ctx = new ClassPathXmlApplicationContext("classpath:s3-valid-inbound-cases.xml");
SourcePollingChannelAdapter valid = ctx.getBean("validInbound", SourcePollingChannelAdapter.class);
AmazonS3InboundSynchronizationMessageSource source = getPropertyValue(valid, "source", AmazonS3InboundSynchronizationMessageSource.class);
assertEquals("TestBucket", getPropertyValue(source, "bucket"));
assertEquals(".temp", getPropertyValue(source, "temporarySuffix"));
assertEquals(new File("/"), getPropertyValue(source, "directory"));
assertEquals("remote", getPropertyValue(source, "remoteDirectory"));
assertEquals(true, getPropertyValue(source, "acceptSubFolders", Boolean.class).booleanValue());
assertEquals(100, getPropertyValue(source, "maxObjectsPerBatch", Integer.class).intValue());
assertEquals("[A-Za-z0-9]+\\\\.txt", getPropertyValue(source, "fileNameRegex"));
//test the second definition with custom attributes
valid = ctx.getBean("validInboundWithCustomOps", SourcePollingChannelAdapter.class);
source = getPropertyValue(valid, "source", AmazonS3InboundSynchronizationMessageSource.class);
assertEquals(new File(System.getProperty("java.io.tmpdir")), getPropertyValue(source, "directory"));
AmazonS3Operations s3Operations = getPropertyValue(source, "s3Operations", AmazonS3Operations.class);
assertEquals(AmazonS3DummyOperations.class, s3Operations.getClass());
//test with aws endpoint set
valid = ctx.getBean("withAWSEndpoint", SourcePollingChannelAdapter.class);
source = getPropertyValue(valid, "source", AmazonS3InboundSynchronizationMessageSource.class);
s3Operations = getPropertyValue(source, "s3Operations", AmazonS3Operations.class);
assertEquals(DefaultAmazonS3Operations.class, s3Operations.getClass());
assertEquals("https://s3-eu-west-1.amazonaws.com", getPropertyValue(s3Operations, "client.endpoint", URI.class).toString());
ctx.close();
}
/**
* Tests with a definition where none of directory and directory-expression attributes are provided
*/
@Test(expected=BeanDefinitionStoreException.class)
public void withNoneOfDirectoryExprAndDirectory() {
new ClassPathXmlApplicationContext("classpath:s3-with-none-of-direxpr-and-dir.xml");
}
}

629
src/test/java/org/springframework/integration/aws/s3/core/AbstractAmazonS3OperationsImplAWSTests.java
View File

@@ -1,629 +0,0 @@
/*
* Copyright 2002-2015 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.aws.s3.core;
import static org.springframework.integration.aws.common.AWSTestUtils.getCredentials;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Set;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TemporaryFolder;
import org.springframework.integration.aws.core.AWSCredentials;
import org.springframework.integration.aws.core.PropertiesAWSCredentials;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.AccessControlList;
import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.Grant;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.S3Object;
/**
* The abstract test class for testing all the common functionality for AWS operations
* on S3 using the appropriate implementation provided by the subclass.
*
* Note: To run the test, you will have to create one bucket for yourself and
* set the name in the {@link #BUCKET_NAME}
*
* @author Amol Nayak
*
* @since 0.5
*
*/
@Ignore()
public abstract class AbstractAmazonS3OperationsImplAWSTests {
private static final String VALID_CANONICAL_ID = "f854da004ee08cf4f8664334d288561c8512c508db9785388de7319ded85f8f3";
@Rule
public final TemporaryFolder temp = new TemporaryFolder();
//private static final String UPLOAD_SOURCE_DIRECTORY = System.getProperty("java.io.tmpdir") + "upload";
//To run the test, you will have to create one one bucket for yourself and
//set the name here
protected static final String BUCKET_NAME = "com.si.aws.test.bucket";
private static AmazonS3Client client;
private static PropertiesAWSCredentials credentials;
@BeforeClass
public static void setup() throws Exception {
AWSCredentials credentials = getCredentials();
client = new AmazonS3Client(
new BasicAWSCredentials(credentials.getAccessKey(), credentials.getSecretKey()));
}
/**
* Sets the multipart threshold value of the upload to 5K,
* this should get executed successfully
*/
@Test
public void withMultipartThresholdWith5k() {
AbstractAmazonS3Operations impl = getS3OperationsImplementation();
impl.setMultipartUploadThreshold(5120);
Assert.assertEquals(5120,impl.getMultipartUploadThreshold());
}
/**
* Sets the multipart threshold value of the upload to a value < 5K,
* should throw IllegalArgumentException
*/
@Test(expected=IllegalArgumentException.class)
public void withMultipartThresholdWithLt5k() {
getS3OperationsImplementation().setMultipartUploadThreshold(5000);
}
/**
* Sets the directory path as a null value.
*/
@Test(expected=IllegalArgumentException.class)
public void setNullDirectoryString() {
getS3OperationsImplementation().setTemporaryDirectory((String)null);
}
/**
* Pass a non null string that exists
*/
@Test
public void setNonNullDirectoryString() {
//this will exist
String directory = System.getProperty("java.io.tmpdir");
getS3OperationsImplementation().setTemporaryDirectory(directory);
}
/**
* Pass a String to a directory that doesn't exist
*/
@Test(expected=IllegalArgumentException.class)
public void setNonExistentDirectory() {
//getting the current time in millis and hope no folder with that name exists
long current = System.currentTimeMillis();
DefaultAmazonS3Operations s3Service = new DefaultAmazonS3Operations(credentials);
s3Service.setTemporaryDirectory("./" + current);
}
/**
* Sets the temporary file suffix to the null value
*/
@Test(expected=IllegalArgumentException.class)
public void setNullTemporaryFileSuffix() {
getS3OperationsImplementation().setTemporaryFileSuffix(null);
}
/**
* Sets the temporary file suffix to a string that begins with a "."
*/
@Test
public void setValidTempFileSuffixStartingWithDot() {
AbstractAmazonS3Operations impl = getS3OperationsImplementation();
impl.setTemporaryFileSuffix(".tempsuff");
Assert.assertEquals(".tempsuff", impl.getTemporaryFileSuffix());
}
/**
* Sets the temporary file suffix to a string that does not begin with a "."
*/
@Test
public void setValidTempFileSuffixStartingWithoutDot() {
AbstractAmazonS3Operations impl = getS3OperationsImplementation();
impl.setTemporaryFileSuffix("tmpsuff");
Assert.assertEquals(".tmpsuff", impl.getTemporaryFileSuffix());
}
//TODO. Test all the conditions that test the folder generation logic, null folder
//null bucket etc
/**
* The AWS Service test put a file with null bucket given
*
*/
public void putToNullBucket() {
AbstractAmazonS3Operations impl = getS3OperationsImplementation();
impl.putObject(null, "/", "name", null);
}
//TODO: Execute the following cases for putObject
/**
* Executes the put object with a null bucket name, should throw
* an {@link IllegalArgumentException}
*/
@Test(expected=IllegalArgumentException.class)
public void putWithNullBucket() {
AmazonS3Operations impl = getS3OperationsImplementation();
impl.putObject(null, null, "SomeObject", null);
}
/**
* Executes the put object with a null object name, should throw
* an {@link IllegalArgumentException}
*/
@Test(expected=IllegalArgumentException.class)
public void withNullObjectName() {
AmazonS3Operations impl = getS3OperationsImplementation();
impl.putObject(BUCKET_NAME, "/", null, null);
}
/**
* Executes the put object with a null s3 object, should throw
* an {@link IllegalArgumentException}
*/
@Test(expected=IllegalArgumentException.class)
public void withNullS3Object() {
AmazonS3Operations impl = getS3OperationsImplementation();
impl.putObject(BUCKET_NAME, "/", "TestObjectName.txt", null);
}
/**
* Executes the put object with both file source and input stream provided
*
* an {@link IllegalArgumentException}
*/
@Test(expected=IllegalArgumentException.class)
public void withBothFileSourceAndInputStream() throws Exception {
String folder = temp.getRoot().getAbsolutePath();
File file = new File(folder + File.separator + "SomeTestFile.txt");
file.createNewFile();
FileInputStream fin = new FileInputStream(file);
new AmazonS3Object(null, null,fin, file);
fin.close();
file.delete();
}
/**
* Executes the put object with none of file source and input stream provided
*
* an {@link IllegalArgumentException}
*/
@Test(expected=IllegalArgumentException.class)
public void withNoneOfFileSourceAndInputStream() throws Exception {
new AmazonS3Object(null, null, null, null);
}
/**
* With a temp file, upload to the bucket and see temp file gets deleted and the object
* successfully uploaded. Also the ACL of the provided object is null, so the
* object should get default ACLs
*
*/
@Test
public void putFromTempFile() throws Exception {
//first delete the file from the AWS bucket
String key = "TestPutFromTempFile.txt";
deleteObject(BUCKET_NAME, key);
File file = generateUploadFile();
//Now put the object
AbstractAmazonS3Operations operations = getS3OperationsImplementation();
operations.setTemporaryDirectory(temp.getRoot());
FileInputStream fin = new FileInputStream(file);
//No ACL associated
AmazonS3Object object = new AmazonS3Object(null, null,fin, null);
operations.putObject(BUCKET_NAME, null, key, object);
assertTempFileDeletion(temp.getRoot().getAbsolutePath(), key);
assertObjectExistenceInBucket(key);
fin.close();
//delete the source file.
file.delete();
}
/**
* Upload a file to a folder in the given bucket with the folder name
* ending with a slash. Also the ACL of the provided object is null, so the
* object should get default ACLs.
*
*/
@Test
public void putToFolderWithEndingSlash() throws Exception {
String uploadFileName = "TestPutWithEndingSlash.txt";
String key = "somedir/with/endingslash/" + uploadFileName;
//first delete the file from the AWS bucket
deleteObject(BUCKET_NAME, key);
File file = generateUploadFile();
//Now put the object
AmazonS3Operations operations = getS3OperationsImplementation();
//No ACL associated
AmazonS3Object object = new AmazonS3Object(null, null, null, file);
operations.putObject(BUCKET_NAME, "somedir/with/endingslash/", uploadFileName, object);
assertObjectExistenceInBucket(key);
//delete the source file.
file.delete();
}
/**
* Upload a file to a folder in the given bucket with the folder name
* ending without a slash. Also the ACL of the provided object is null, so the
* object should get default ACLs.
*
*/
@Test
public void putToFolderWithoutEndingSlash() throws Exception {
String uploadFile = "TestPutWithoutEndingSlash.txt";
String key = "somedir/without/endingslash/" + uploadFile;
//first delete the file from the AWS bucket
deleteObject(BUCKET_NAME, key);
File file = generateUploadFile();
//Now put the object
AmazonS3Operations operations = getS3OperationsImplementation();
//No ACL associated
AmazonS3Object object = new AmazonS3Object(null, null, null, file);
operations.putObject(BUCKET_NAME, "somedir/without/endingslash", uploadFile, object);
assertObjectExistenceInBucket(key);
//delete the source file.
file.delete();
}
/**
* Upload a file to a folder in the given bucket with the folder name
* beginning a slash. Also the ACL of the provided object is null, so the
* object should get default ACLs.
*
*/
@Test
public void putToFolderBeginningWithSlash() throws Exception {
String uploadFileName = "TestPutBeginningWithSlash.txt";
String key = "beginning/with/slash/" + uploadFileName;
//first delete the file from the AWS bucket
deleteObject(BUCKET_NAME, key);
File file = generateUploadFile();
//Now put the object
AmazonS3Operations operations = getS3OperationsImplementation();
//No ACL associated
AmazonS3Object object = new AmazonS3Object(null, null, null, file);
operations.putObject(BUCKET_NAME, "/beginning/with/slash/", uploadFileName, object);
assertObjectExistenceInBucket(key);
//delete the source file.
file.delete();
}
/**
* Upload a file with the provided ACLs and meta data, the test verifies if the
* ACLs and the metadata of the file is appropriately set
*
*/
@Test
public void putToFolderForACLAndMetadataTest() throws Exception {
String uploadFileName = "TestObjectACLAndMetaData.txt";
String key = "acl/and/metadata/test/" + uploadFileName;
//first delete the file from the AWS bucket
deleteObject(BUCKET_NAME, key);
File file = generateUploadFile();
FileInputStream fin = new FileInputStream(file);
//Now put the object
AmazonS3Operations operations = getS3OperationsImplementation();
Map<String, String> userMetaData = Collections.singletonMap("TestKey", "TestValue");
AmazonS3ObjectACL acl = new AmazonS3ObjectACL();
ObjectGrant grant = new ObjectGrant(new Grantee(VALID_CANONICAL_ID, GranteeType.CANONICAL_GRANTEE_TYPE),
ObjectPermissions.READ_ACP);
acl.addGrant(grant);
AmazonS3Object object = new AmazonS3Object(userMetaData, null, fin, null,acl);
operations.putObject(BUCKET_NAME, "/acl/and/metadata/test/", uploadFileName, object);
//This fails somehow on my machine
//assertTempFileDeletion(uploadFileName);
//NOTE: The case of the key is no longer in the case we used, its all lower case.
//lets get the object's User metadata first
S3Object s3Object = getObject(BUCKET_NAME,key);
ObjectMetadata objectMetadata = s3Object.getObjectMetadata();
userMetaData = objectMetadata.getUserMetadata();
Assert.assertNotNull("User metadata is not expected to be null, but got null", userMetaData);
Assert.assertTrue("Expecting the key 'testkey' in user MetaData", userMetaData.containsKey("testkey"));
Assert.assertEquals("TestValue", userMetaData.get("testkey"));
//lets verify the object's ACL
AccessControlList acls = getObjectACL(BUCKET_NAME, key);
List<Grant> grants = acls.getGrantsAsList();
boolean isACLValid = false;
for(Grant g : grants) {
com.amazonaws.services.s3.model.Grantee grantee = g.getGrantee();
if(VALID_CANONICAL_ID.equals(grantee.getIdentifier())
&& "READ_ACP".equals(grant.getPermission().toString())) {
isACLValid = true;
}
}
Assert.assertTrue("Expected Object ACl not found", isACLValid);
fin.close();
//delete the source file.
file.delete();
}
/**
* List the contents in the bucket with null bucket name
*/
@Test(expected=IllegalArgumentException.class)
public void listWithNullBucket() {
AmazonS3Operations impl = getS3OperationsImplementation();
impl.listObjects(null, "folder", null, 1);
}
/**
* List objects with negative page size
*/
@Test(expected=IllegalArgumentException.class)
public void listWithPageSizeLt0() {
AmazonS3Operations impl = getS3OperationsImplementation();
impl.listObjects(BUCKET_NAME, "folder", null, -2);
}
/**
* List with null folder
*/
@Test
public void listWithNullFolder() {
AmazonS3Operations impl = getS3OperationsImplementation();
PaginatedObjectsView pov = impl.listObjects(BUCKET_NAME, null, null, 100);
List<S3ObjectSummary> summary = pov.getObjectSummary();
Assert.assertNotNull(summary);
Assert.assertTrue(summary.size() > 0);
System.out.println("Summary list size is " + summary.size());
}
/**
* List with folder as a slash(/), for root folder
*/
@Test
public void listWithSlashOnRoot() {
AmazonS3Operations impl = getS3OperationsImplementation();
PaginatedObjectsView pov = impl.listObjects(BUCKET_NAME, "/", null, 100);
List<S3ObjectSummary> summary = pov.getObjectSummary();
Assert.assertNotNull(summary);
Assert.assertTrue(summary.size() > 0);
System.out.println("Summary list size is " + summary.size());
}
/**
* List with folder as a slash(/)
*/
@Test
public void listWithFolderAsSlash() {
AmazonS3Operations impl = getS3OperationsImplementation();
PaginatedObjectsView pov = impl.listObjects(BUCKET_NAME, "/acl", null, 100);
List<S3ObjectSummary> summary = pov.getObjectSummary();
Assert.assertNotNull(summary);
Assert.assertTrue(summary.size() > 0);
System.out.println("Summary list size is " + summary.size());
}
/**
* List with folder as a not beginning with slash(/)
*/
@Test
public void listWithFolderNotBeginningWithSlash() {
AmazonS3Operations impl = getS3OperationsImplementation();
PaginatedObjectsView pov = impl.listObjects(BUCKET_NAME, "somedir/with", null, 100);
List<S3ObjectSummary> summary = pov.getObjectSummary();
Assert.assertNotNull(summary);
Assert.assertTrue(summary.size() > 0);
System.out.println("Summary list size is " + summary.size());
}
/**
* The test case assumes that all previous AWS tests are executed and we have at least 4 objects
* in the bucket, on running all the above tests you will have 5, so we need not do anything
* special to add more objects to execute this test
*
*/
@Test
public void paginateRecords() {
AmazonS3Operations impl = getS3OperationsImplementation();
PaginatedObjectsView pov = impl.listObjects(BUCKET_NAME, "/", null, 3);
List<S3ObjectSummary> summary = pov.getObjectSummary();
Assert.assertNotNull(summary);
Assert.assertEquals(3, summary.size());
String nextMarker = pov.getNextMarker();
Assert.assertNotNull("Expected a non null marker", nextMarker);
pov = impl.listObjects(BUCKET_NAME, "/", nextMarker, 3);
summary = pov.getObjectSummary();
Assert.assertNotNull(summary);
Assert.assertTrue(summary.size() > 0);
System.out.printf("Number of records on second page are %d\n",summary.size());
}
/**
* Tests the get object with a null bucket
*/
@Test(expected=IllegalArgumentException.class)
public void getObjectFromNullBucket() {
AmazonS3Operations impl = getS3OperationsImplementation();
impl.getObject(null, null, "ObjectName.txt");
}
/**
* Gets a non existent object from the bucket, should return null on unsuccessful search
* and if the object with the key doesn't exist.
*/
@Test
public void getNonExistentObject() {
AmazonS3Operations impl = getS3OperationsImplementation();
AmazonS3Object object = impl.getObject(BUCKET_NAME, null, "jhgkmjbhdc.thb");
Assert.assertNull("Expecting a null object but got a non null one", object);
}
/**
* Invoked the getObject with null folder, this will get the object
* from the root of the bucket.
*
*/
@Test
public void getObjectWithNullFolder() {
AmazonS3Operations impl = getS3OperationsImplementation();
AmazonS3Object object = impl.getObject(BUCKET_NAME, null, "TestPutFromTempFile.txt");
Assert.assertNotNull("Expecting a non null object but got a null one", object);
}
/**
* Invoked the getObject with folder name beginning with /
*
*/
@Test
public void getObjectromFolderBeginningWithSlash() {
AmazonS3Operations impl = getS3OperationsImplementation();
AmazonS3Object object = impl.getObject(BUCKET_NAME, "/acl/and/metadata/test", "TestObjectACLAndMetaData.txt");
Assert.assertNotNull("Expecting a non null object but got a null one", object);
}
/**
* Invoked the getObject with folder name beginning with /
*
*/
@Test
public void getObjectFromFolderBeginningWithoutSlash() {
AmazonS3Operations impl = getS3OperationsImplementation();
AmazonS3Object object = impl.getObject(BUCKET_NAME, "acl/and/metadata/test", "TestObjectACLAndMetaData.txt");
Assert.assertNotNull("Expecting a non null object but got a null one", object);
}
/**
* The common method that checks if the temp file generated for the
* test file uploaded is deleted.
*/
private void assertTempFileDeletion(String rootFolder, String baseFileName) {
//Check if the temp file exists.
File tempFile = new File(rootFolder + File.separator + baseFileName + ".writing");
Assert.assertFalse("Was expecting the temp file to be deleted, but is present", tempFile.exists());
}
/**
* Common method that will assert the existence of the object with the given key in the
* bucket
*
* @param key
*/
private void assertObjectExistenceInBucket(String key) {
S3Object s3Object = getObject(BUCKET_NAME, key);
//This is not needed as an exception will be thrown if the key does not exist
Assert.assertNotNull("Non null S3Object expected",s3Object);
}
/**
* The private helper method that generates the test file to be uploaded
* @throws IOException
* @throws FileNotFoundException
*/
private File generateUploadFile() throws IOException, FileNotFoundException {
//TODO: Move this to @BeforeClass?
String fileName = System.currentTimeMillis() + ".txt";
File file = new File(temp.getRoot().getAbsolutePath() + File.separator + fileName);
file.createNewFile();
//Write something to it
FileOutputStream fos = new FileOutputStream(file);
fos.write("Test".getBytes());
fos.close();
return file;
}
//-- helper methods to interact with AWS S3 services
//These methods are used to verify and assert if the implementation
//has performed the desired operation
/**
* Gets the object from the S3 bucket
*
* @param gets the object from the bucket with the given key
*/
protected S3Object getObject(String bucket,String key) {
return client.getObject(new GetObjectRequest(bucket, key));
}
/**
* Gets the objects's {@link AccessControlList} (ACL) for the given bucket and key
*
* @param bucket
* @param key
*/
protected AccessControlList getObjectACL(String bucket,String key) {
return client.getObjectAcl(bucket, key);
}
/**
* Deleted the given object name from the given bucket and key
*
* @param bucket
* @param key
*
*/
protected void deleteObject(String bucket,String key) {
client.deleteObject(bucket, key);
}
protected abstract AbstractAmazonS3Operations getS3OperationsImplementation();
}

87
src/test/java/org/springframework/integration/aws/s3/core/DefaultAmazonS3OperationsAWSTests.java
View File

@@ -1,87 +0,0 @@
/*
* Copyright 2002-2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.integration.aws.s3.core;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadPoolExecutor;
import org.junit.Assert;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
import org.springframework.integration.aws.core.PropertiesAWSCredentials;
/**
* The test class for the {@link DefaultAmazonS3Operations}, the default implementation that
* uses the AWS SDK to implement the functionality. The tests are present in the superclass
* {@link AbstractAmazonS3OperationsImplAWSTests}
*
* Please note that that this test needs connectivity with the AWS S3 service
* to be successfully executed. It is excluded from the maven's test execution by default
*
* To run this test you need to have your AWSAccess key and Secret key in the
* file awscredentials.properties in the classpath. This file is not present in the
* repository and you need to add one yourselves to src/test/resources folder and have
* two properties accessKey and secretKey in it containing the access and the secret key
*
*
* @author Amol Nayak
*
* @since 0.5
*
*/
@Ignore
public class DefaultAmazonS3OperationsAWSTests extends AbstractAmazonS3OperationsImplAWSTests {
private static DefaultAmazonS3Operations impl;
@BeforeClass
public static void setupS3Operations() throws Exception {
PropertiesAWSCredentials credentials =
new PropertiesAWSCredentials("classpath:awscredentials.properties");
credentials.afterPropertiesSet();
impl = new DefaultAmazonS3Operations(credentials);
}
/**
* Sets the thread pool executor to a non null value, execution should
* complete successfully
*
*/
@Ignore
@Test
public void withNonNullThreadPoolExecutor() {
ThreadPoolExecutor executor = (ThreadPoolExecutor)Executors.newFixedThreadPool(10);
impl.setThreadPoolExecutor(executor);
Assert.assertEquals(executor, impl.getThreadPoolExecutor());
}
/**
* Sets the thread pool executor to a null value, should throw an
* {@link IllegalArgumentException}
*/
@Ignore
@Test(expected=IllegalArgumentException.class)
public void withNullThreadPoolExecutor() {
impl.setThreadPoolExecutor(null);
}
@Override
protected AbstractAmazonS3Operations getS3OperationsImplementation() {
return impl;
}
}