diff --git a/README.md b/README.md index 8522701..52efb14 100644 --- a/README.md +++ b/README.md @@ -65,6 +65,77 @@ Get more information about AWS free tier at [http://aws.amazon.com/free/][]** The S3 Channel Adapters are based on the `AmazonS3` template and `TransferManager`. See their specification and JavaDocs for more information. +###Inbound Channel Adapter + +The S3 Inbound Channel Adapter is represented by the `S3InboundFileSynchronizingMessageSource` + (``) and allows to pull S3 objects as files from the S3 bucket +to the local directory for synchronization. +This adapter is fully similar to the Inbound Channel Adapters in the FTP and SFTP Spring Integration modules. +See more information in the [FTP/FTPS Adapters Chapter][] for common options or `SessionFactory`, `RemoteFileTemplate` +and `FileListFilter` abstractions. + +The Java Configuration is: + +````java +@SpringBootApplication +public static class MyConfiguration { + + @Autowired + private AmazonS3 amazonS3; + + @Bean + public S3InboundFileSynchronizer s3InboundFileSynchronizer() { + S3InboundFileSynchronizer synchronizer = new S3InboundFileSynchronizer(amazonS3()); + synchronizer.setDeleteRemoteFiles(true); + synchronizer.setPreserveTimestamp(true); + synchronizer.setRemoteDirectory(S3_BUCKET); + synchronizer.setFilter(new S3RegexPatternFileListFilter(".*\\.test$")); + Expression expression = PARSER.parseExpression("#this.toUpperCase() + '.a'"); + synchronizer.setLocalFilenameGeneratorExpression(expression); + return synchronizer; + } + + @Bean + @InboundChannelAdapter(value = "s3FilesChannel", poller = @Poller(fixedDelay = "100")) + public S3InboundFileSynchronizingMessageSource s3InboundFileSynchronizingMessageSource() { + S3InboundFileSynchronizingMessageSource messageSource = + new S3InboundFileSynchronizingMessageSource(s3InboundFileSynchronizer()); + messageSource.setAutoCreateLocalDirectory(true); + messageSource.setLocalDirectory(LOCAL_FOLDER); + messageSource.setLocalFilter(new AcceptOnceFileListFilter()); + return messageSource; + } + + @Bean + public PollableChannel s3FilesChannel() { + return new QueueChannel(); + } +} +```` + +With this config you receive messages with `java.io.File` `payload` from the `s3FilesChannel` +after periodic synchronization of content from the Amazon S3 bucket into the local directory. + +An XML variant may look like: + +````xml + + + + + +```` + ###Outbound Channel Adapter The S3 Outbound Channel Adapter is represented by the `S3MessageHandler` (`` @@ -366,3 +437,4 @@ By default the `SnsMessageHandler` is one-way `MessageHandler`. [http://aws.amazon.com/ses/]: http://aws.amazon.com/ses/ [http://aws.amazon.com/documentation/ses/]: http://aws.amazon.com/documentation/ses/ [http://aws.amazon.com/free/]: http://aws.amazon.com/free/ +[Reference Manual]: http://docs.spring.io/spring-integration/reference/html/ftp.html diff --git a/build.gradle b/build.gradle index 02c312a..8feecdc 100644 --- a/build.gradle +++ b/build.gradle @@ -1,5 +1,3 @@ -description = 'Spring Integration AWS Support' - buildscript { repositories { maven { url 'http://repo.spring.io/plugins-release' } @@ -9,6 +7,12 @@ buildscript { } } +plugins { + id "org.sonarqube" version "1.2" +} + +description = 'Spring Integration AWS Support' + apply plugin: 'java' apply from: "${rootProject.projectDir}/publish-maven.gradle" apply plugin: 'eclipse' @@ -39,9 +43,8 @@ if (project.hasProperty('platformVersion')) { } ext { - commonsIoVersion='2.4' servletApiVersion = '3.1.0' - slf4jVersion = '1.7.13' + slf4jVersion = '1.7.18' springCloudAwsVersion = '1.1.0.M2' springIntegrationVersion = '4.2.5.RELEASE' @@ -60,16 +63,16 @@ ext.javadocLinks = [ ] as String[] jacoco { - toolVersion = "0.7.2.201409121644" + toolVersion = "0.7.6.201602180812" } dependencies { - compile "org.springframework.integration:spring-integration-core:$springIntegrationVersion" - compile ("org.springframework.integration:spring-integration-http:$springIntegrationVersion", optional) - compile("javax.servlet:javax.servlet-api:$servletApiVersion", provided) compile "org.springframework.cloud:spring-cloud-aws-messaging:$springCloudAwsVersion" - compile "commons-io:commons-io:$commonsIoVersion" + compile "org.springframework.integration:spring-integration-core:$springIntegrationVersion" + compile ("org.springframework.integration:spring-integration-file:$springIntegrationVersion", optional) + compile ("org.springframework.integration:spring-integration-http:$springIntegrationVersion", optional) + compile ("javax.servlet:javax.servlet-api:$servletApiVersion", provided) testCompile "org.springframework.integration:spring-integration-test:$springIntegrationVersion" @@ -151,10 +154,8 @@ artifacts { archives javadocJar } -apply plugin: 'sonar-runner' - -sonarRunner { - sonarProperties { +sonarqube { + properties { property "sonar.jacoco.reportPath", "${buildDir.name}/jacoco.exec" property "sonar.links.homepage", linkHomepage property "sonar.links.ci", linkCi @@ -275,9 +276,3 @@ task dist(dependsOn: assemble) { group = 'Distribution' description = 'Builds -dist, -docs and -schema distribution archives.' } - -task wrapper(type: Wrapper) { - description = 'Generates gradlew[.bat] scripts' - gradleVersion = '2.5' - distributionUrl = "http://services.gradle.org/distributions/gradle-${gradleVersion}-all.zip" -} diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 30d399d..2c6137b 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index 741e43d..c280f02 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,6 +1,6 @@ -#Wed Sep 23 15:22:04 EDT 2015 +#Thu Mar 31 14:54:33 EDT 2016 distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists -distributionUrl=http\://services.gradle.org/distributions/gradle-2.5-all.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-2.12-bin.zip diff --git a/gradlew b/gradlew index 91a7e26..9d82f78 100755 --- a/gradlew +++ b/gradlew @@ -42,11 +42,6 @@ case "`uname`" in ;; esac -# For Cygwin, ensure paths are in UNIX format before anything is touched. -if $cygwin ; then - [ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --unix "$JAVA_HOME"` -fi - # Attempt to set APP_HOME # Resolve links: $0 may be a link PRG="$0" @@ -61,9 +56,9 @@ while [ -h "$PRG" ] ; do fi done SAVED="`pwd`" -cd "`dirname \"$PRG\"`/" >&- +cd "`dirname \"$PRG\"`/" >/dev/null APP_HOME="`pwd -P`" -cd "$SAVED" >&- +cd "$SAVED" >/dev/null CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar @@ -114,6 +109,7 @@ fi if $cygwin ; then APP_HOME=`cygpath --path --mixed "$APP_HOME"` CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + JAVACMD=`cygpath --unix "$JAVACMD"` # We build the pattern for arguments to be converted via cygpath ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` diff --git a/gradlew.bat b/gradlew.bat index aec9973..72d362d 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -46,7 +46,7 @@ echo location of your Java installation. goto fail :init -@rem Get command-line arguments, handling Windowz variants +@rem Get command-line arguments, handling Windows variants if not "%OS%" == "Windows_NT" goto win9xME_args if "%@eval[2+2]" == "4" goto 4NT_args diff --git a/src/main/java/org/springframework/integration/aws/config/xml/AwsNamespaceHandler.java b/src/main/java/org/springframework/integration/aws/config/xml/AwsNamespaceHandler.java index 45d40ad..285ad42 100644 --- a/src/main/java/org/springframework/integration/aws/config/xml/AwsNamespaceHandler.java +++ b/src/main/java/org/springframework/integration/aws/config/xml/AwsNamespaceHandler.java @@ -16,7 +16,6 @@ package org.springframework.integration.aws.config.xml; -import org.springframework.integration.aws.s3.config.xml.AmazonS3InboundChannelAdapterParser; import org.springframework.integration.config.xml.AbstractIntegrationNamespaceHandler; /** @@ -32,7 +31,7 @@ public class AwsNamespaceHandler extends AbstractIntegrationNamespaceHandler { public void init() { registerBeanDefinitionParser("s3-outbound-channel-adapter", new S3OutboundChannelAdapterParser()); registerBeanDefinitionParser("s3-outbound-gateway", new S3OutboundGatewayParser()); - registerBeanDefinitionParser("s3-inbound-channel-adapter", new AmazonS3InboundChannelAdapterParser()); + registerBeanDefinitionParser("s3-inbound-channel-adapter", new S3InboundChannelAdapterParser()); registerBeanDefinitionParser("sqs-outbound-channel-adapter", new SqsOutboundChannelAdapterParser()); registerBeanDefinitionParser("sqs-message-driven-channel-adapter", new SqsMessageDrivenChannelAdapterParser()); registerBeanDefinitionParser("sns-inbound-channel-adapter", new SnsInboundChannelAdapterParser()); diff --git a/src/main/java/org/springframework/integration/aws/config/xml/AwsParserUtils.java b/src/main/java/org/springframework/integration/aws/config/xml/AwsParserUtils.java index 8b73a89..8dfede5 100644 --- a/src/main/java/org/springframework/integration/aws/config/xml/AwsParserUtils.java +++ b/src/main/java/org/springframework/integration/aws/config/xml/AwsParserUtils.java @@ -1,5 +1,5 @@ /* - * Copyright 2002-2015 the original author or authors. + * Copyright 2002-2016 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -16,16 +16,6 @@ package org.springframework.integration.aws.config.xml; -import org.w3c.dom.Element; - -import org.springframework.beans.factory.support.BeanDefinitionBuilder; -import org.springframework.beans.factory.support.BeanDefinitionReaderUtils; -import org.springframework.beans.factory.xml.ParserContext; -import org.springframework.integration.aws.core.AWSCredentials; -import org.springframework.integration.aws.core.BasicAWSCredentials; -import org.springframework.integration.aws.core.PropertiesAWSCredentials; -import org.springframework.util.StringUtils; - /** * The utility class for the namespace parsers * @@ -36,14 +26,6 @@ import org.springframework.util.StringUtils; */ public final class AwsParserUtils { - public static final String ACCESS_KEY = "accessKey"; - - public static final String SECRET_KEY = "secretKey"; - - public static final String PROPERTIES_FILE = "propertiesFile"; - - public static final String CREDENTIALS_REF = "credentials-ref"; - public static final String SQS_REF = "sqs"; public static final String SNS_REF = "sns"; @@ -53,59 +35,7 @@ public final class AwsParserUtils { public static final String RESOURCE_ID_RESOLVER_REF = "resource-id-resolver"; private AwsParserUtils() { - } - - - /** - * Registers the {@link AWSCredentials} bean with the current ApplicationContext if - * accessKey and secretKey is given, if the credentials-ref is given, the given value - * is returned. - * @param element - * @param parserContext - */ - public static String getAmazonWSCredentials(Element element, ParserContext parserContext) { - //TODO: Some mechanism to use the same instance with same ACCESS_KEY to be implemented - String accessKey = element.getAttribute(ACCESS_KEY); - String secretKey = element.getAttribute(SECRET_KEY); - String propertiesFile = element.getAttribute(PROPERTIES_FILE); - String credentialsRef = element.getAttribute(CREDENTIALS_REF); - String awsCredentialsGeneratedName; - - if (StringUtils.hasText(credentialsRef)) { - if (StringUtils.hasText(propertiesFile) - || StringUtils.hasText(accessKey) - || StringUtils.hasText(secretKey)) { - parserContext.getReaderContext().error("When " + CREDENTIALS_REF + " is specified, " + - "do not specify the " + PROPERTIES_FILE + " attribute or the " - + SECRET_KEY + " and " + ACCESS_KEY + " attributes", element); - } - awsCredentialsGeneratedName = credentialsRef; - } - else { - if (StringUtils.hasText(propertiesFile)) { - if (StringUtils.hasText(accessKey) && StringUtils.hasText(secretKey)) { - parserContext.getReaderContext().error("When " + ACCESS_KEY + " and " + SECRET_KEY + - " are specified, do not specify the " + PROPERTIES_FILE + " attribute", element); - } - - BeanDefinitionBuilder builder = - BeanDefinitionBuilder.genericBeanDefinition(PropertiesAWSCredentials.class); - builder.addConstructorArgValue(propertiesFile); - awsCredentialsGeneratedName = BeanDefinitionReaderUtils.registerWithGeneratedName( - builder.getBeanDefinition(), parserContext.getRegistry()); - } - else { - BeanDefinitionBuilder builder - = BeanDefinitionBuilder.genericBeanDefinition(BasicAWSCredentials.class); - builder.addConstructorArgValue(accessKey); - builder.addConstructorArgValue(secretKey); - awsCredentialsGeneratedName = BeanDefinitionReaderUtils.registerWithGeneratedName( - builder.getBeanDefinition(), parserContext.getRegistry()); - - } - } - - return awsCredentialsGeneratedName; + super(); } } diff --git a/src/main/java/org/springframework/integration/aws/config/xml/S3InboundChannelAdapterParser.java b/src/main/java/org/springframework/integration/aws/config/xml/S3InboundChannelAdapterParser.java new file mode 100644 index 0000000..6c48348 --- /dev/null +++ b/src/main/java/org/springframework/integration/aws/config/xml/S3InboundChannelAdapterParser.java @@ -0,0 +1,52 @@ +/* + * Copyright 2016 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.aws.config.xml; + +import org.springframework.integration.aws.inbound.S3InboundFileSynchronizer; +import org.springframework.integration.aws.inbound.S3InboundFileSynchronizingMessageSource; +import org.springframework.integration.aws.support.filters.S3RegexPatternFileListFilter; +import org.springframework.integration.aws.support.filters.S3SimplePatternFileListFilter; +import org.springframework.integration.file.config.AbstractRemoteFileInboundChannelAdapterParser; + +/** + * Parser for the AWS 's3-inbound-channel-adapter' element. + * + * @author Artem Bilan + */ +public class S3InboundChannelAdapterParser extends AbstractRemoteFileInboundChannelAdapterParser { + + @Override + protected String getMessageSourceClassname() { + return S3InboundFileSynchronizingMessageSource.class.getName(); + } + + @Override + protected String getInboundFileSynchronizerClassname() { + return S3InboundFileSynchronizer.class.getName(); + } + + @Override + protected String getSimplePatternFileListFilterClassname() { + return S3SimplePatternFileListFilter.class.getName(); + } + + @Override + protected String getRegexPatternFileListFilterClassname() { + return S3RegexPatternFileListFilter.class.getName(); + } + +} diff --git a/src/main/java/org/springframework/integration/aws/core/AWSClientFactory.java b/src/main/java/org/springframework/integration/aws/core/AWSClientFactory.java deleted file mode 100644 index c76216f..0000000 --- a/src/main/java/org/springframework/integration/aws/core/AWSClientFactory.java +++ /dev/null @@ -1,39 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.aws.core; - -import com.amazonaws.AmazonWebServiceClient; - -/** - * The factory interface that would be used to get the implementation of the appropriate - * instance of {@link AmazonWebServiceClient} - * - * @author Amol Nayak - * - * @since 0.5 - * - */ -public interface AWSClientFactory { - - /** - * Returns the instance of the {@link AmazonWebServiceClient} with the apropriate endpoint value - * set based on the provided url value - * - * @param url The url of the service - * @return The appropriate {@link AmazonWebServiceClient} for the provided endpoint URL - */ - T getClient(String url); -} diff --git a/src/main/java/org/springframework/integration/aws/core/AWSCommonUtils.java b/src/main/java/org/springframework/integration/aws/core/AWSCommonUtils.java deleted file mode 100644 index 845a8f0..0000000 --- a/src/main/java/org/springframework/integration/aws/core/AWSCommonUtils.java +++ /dev/null @@ -1,107 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.aws.core; - -import java.io.BufferedInputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.security.DigestInputStream; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; - -import org.apache.commons.codec.binary.Base64; -import org.apache.commons.codec.binary.Hex; -import org.apache.commons.io.IOUtils; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -/** - * The common utility methods for the - * - * @author Amol Nayak - * - * @since 0.5 - * - */ -public class AWSCommonUtils { - - private static final Log logger = LogFactory.getLog(AWSCommonUtils.class); - - /** - * Generates the MD5 hash of the file provided - * @param file - */ - public static byte[] getContentsMD5AsBytes(File file) { - - DigestInputStream din = null; - final byte[] digestToReturn; - - try { - BufferedInputStream bin = new BufferedInputStream(new FileInputStream(file),32768); - din = new DigestInputStream(bin, MessageDigest.getInstance("MD5")); - //Just to update the digest - byte[] dummy = new byte[4096]; - for (int i = 1; i > 0; i = din.read(dummy)); - digestToReturn = din.getMessageDigest().digest(); - } - catch (NoSuchAlgorithmException e) { - throw new IllegalStateException("Caught Exception while generating a MessageDigest instance", e); - } - catch (FileNotFoundException e) { - throw new IllegalStateException("File " + file.getName() + " not found", e); - } - catch(IOException e) { - throw new IllegalStateException("Caught exception while reading from file", e); - } - finally { - IOUtils.closeQuietly(din); - } - return digestToReturn; - } - - /** - * Compute the MD5 hash of the provided String - * @param contents The String whose MD5 sun is to be computed - */ - public static byte[] getContentsMD5AsBytes(String contents) { - try { - MessageDigest digest = MessageDigest.getInstance("MD5"); - return digest.digest(contents.getBytes()); - } catch (NoSuchAlgorithmException e) { - throw new IllegalStateException(String.format("Unable to digest the input String '%s' using MD5.", contents), e); - } - } - - /** - * Encodes the given raw bytes into hex - * @param rawBytes - */ - public static String encodeHex(byte[] rawBytes) throws UnsupportedEncodingException { - return new String(Hex.encodeHex(rawBytes)); - } - - /** - * Decodes the given base 64 raw bytes - * - * @param rawBytes - */ - public static byte[] decodeBase64(byte[] rawBytes) throws UnsupportedEncodingException { - return Base64.decodeBase64(rawBytes); - } -} \ No newline at end of file diff --git a/src/main/java/org/springframework/integration/aws/core/AWSCredentials.java b/src/main/java/org/springframework/integration/aws/core/AWSCredentials.java deleted file mode 100644 index 0d63bb6..0000000 --- a/src/main/java/org/springframework/integration/aws/core/AWSCredentials.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.aws.core; - -/** - * The common interfaces for all implementations of Amazon WS Credentials - * - * @author Amol Nayak - * - * @since 0.5 - * - */ -public interface AWSCredentials { - - /** - * Get the Access key to the Amazon WS account - */ - String getAccessKey(); - - /** - * Get the Secret key to the Amazon WS account - */ - String getSecretKey(); - -} diff --git a/src/main/java/org/springframework/integration/aws/core/AWSOperationException.java b/src/main/java/org/springframework/integration/aws/core/AWSOperationException.java deleted file mode 100644 index 2cd192f..0000000 --- a/src/main/java/org/springframework/integration/aws/core/AWSOperationException.java +++ /dev/null @@ -1,64 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.aws.core; - -/** - * The Base class for all other AWS operation exceptions - * - * @author Amol Nayak - * - * @since 0.5 - * - */ -public class AWSOperationException extends RuntimeException { - - /** - * - */ - private static final long serialVersionUID = 3391888045993691634L; - - private final String accessKey; - - public AWSOperationException(String accessKey) { - super(); - this.accessKey = accessKey; - } - - public AWSOperationException(String accessKey,String message) { - super(message); - this.accessKey = accessKey; - } - - public AWSOperationException(String accessKey,String message, Throwable cause) { - super(message, cause); - this.accessKey = accessKey; - } - - public AWSOperationException(String accessKey,Throwable cause) { - super(cause); - this.accessKey = accessKey; - } - - /** - * Get the access key for the user who encountered the exception - */ - public String getAccessKey() { - return accessKey; - } - - - -} diff --git a/src/main/java/org/springframework/integration/aws/core/AbstractAWSClientFactory.java b/src/main/java/org/springframework/integration/aws/core/AbstractAWSClientFactory.java deleted file mode 100644 index c2161b9..0000000 --- a/src/main/java/org/springframework/integration/aws/core/AbstractAWSClientFactory.java +++ /dev/null @@ -1,157 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.aws.core; - -import java.net.MalformedURLException; -import java.net.URL; -import java.util.HashMap; -import java.util.Map; -import java.util.concurrent.ConcurrentHashMap; - -import org.springframework.util.StringUtils; - -import com.amazonaws.AmazonWebServiceClient; - - -/** - * The abstract factory class that will be used by all the client operations to acquire - * the appropriate implementation of the {@link AmazonWebServiceClient} based on the URL - * passed to the getClient method - * - * @author Amol Nayak - * - * @since 0.5 - * - */ -public abstract class AbstractAWSClientFactory implements AWSClientFactory { - - /** - * A map storing the {@link AmazonWebServiceClient} endpoint as the key and the SQS Client as the - * value. Since setting the endpoint is not a thread safe operation once the client is instantiated, - * we maintain a map of endpoint and the {@link AmazonWebServiceClient} instantiated the first time a request - * for the designated endpoint is received - */ - private final ConcurrentHashMap clientMap = new ConcurrentHashMap(); - - /** - * The String constant for HTTP - */ - protected final static String HTTP = "http://"; - - /** - * The String constant for HTTPS - */ - protected final static String HTTPS = "https://"; - - /** - * The String constant for SMTP - */ - protected final String SMTP = "smtp://"; - - /** - * The default protocol to be used in case none is provided - */ - protected final static String DEFAULT_PROTOCOL = HTTPS; - - private T defaultEndpointInstance; - - - /** - * Returns the cached implementation of the {@link AmazonWebServiceClient} based on the URL provided. - * the client instance is acquired using the abstract getClientImplementation method. - * The instance is added to the client map with the endpoint string as the key and the - * {@link AmazonWebServiceClient} as the value. - * - * @param url the URL for which the client is requested. - * @return the implementation of the {@link AmazonWebServiceClient} to be used for the provided url - */ - public final T getClient(String url) { - String endpoint = getEndpointFromURL(url); - if(endpoint == null) { - if(defaultEndpointInstance == null) { - defaultEndpointInstance = getClientImplementation(); - } - return defaultEndpointInstance; - } - if(!clientMap.containsKey(endpoint)) { - T client = getClientImplementation(); - client.setEndpoint(endpoint); - T existingClient = clientMap.putIfAbsent(endpoint, client); - if(existingClient != null) { - //in rare scenarios where a new implementation was created after - //checking for the existence of the endpoint in the client map - client = existingClient; - } - return client; - } - else { - return clientMap.get(endpoint); - } - } - - /** - * Return a copy of the client map - * @return the copy of the clientMap - */ - public final Map getClientMap() { - return new HashMap(clientMap); - } - - /** - * Clears the complete cache - */ - public final void clear() { - clientMap.clear(); - } - - /** - * Extracts the endpoint from the URL provided - * - * @return Will return null if the provided url is empty - */ - private String getEndpointFromURL(String stringUrl) { - if(!StringUtils.hasText(stringUrl)) { - return null; - } - String endpoint; - try { - if(!(stringUrl.startsWith(HTTP) - || stringUrl.startsWith(HTTPS) - || stringUrl.startsWith(SMTP))) { - stringUrl = DEFAULT_PROTOCOL + stringUrl; - } - URL url = new URL(stringUrl); - String host = url.getHost(); - String protocol = url.getProtocol(); - if(StringUtils.hasText(protocol)) { - endpoint = protocol + "://" + host; - } - else { - endpoint = host; - } - } catch (MalformedURLException e) { - throw new AWSOperationException(null, "The URL \"" + stringUrl + "\" is malformed",e); - } - return endpoint; - } - - /** - * The subclass needs to implement this method and return an appropriate implementation - */ - protected abstract T getClientImplementation(); - - -} \ No newline at end of file diff --git a/src/main/java/org/springframework/integration/aws/core/BasicAWSCredentials.java b/src/main/java/org/springframework/integration/aws/core/BasicAWSCredentials.java deleted file mode 100644 index 818c506..0000000 --- a/src/main/java/org/springframework/integration/aws/core/BasicAWSCredentials.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.aws.core; - -import org.springframework.util.Assert; - -/** - * The basic implementation class holding the Access key and the secret - * key for the AWS account . - * - * @author Amol Nayak - * - * @since 0.5 - * - */ -public class BasicAWSCredentials implements AWSCredentials { - - /**. - * Hold the Access key for the AWS account - */ - private String accessKey; - - /**. - * Hold the Secret key for the - */ - private String secretKey; - - - /**. - * Default constructor - */ - public BasicAWSCredentials() { - - } - - /**. - * The constructor accepting the access and secret key. - * - * @param accessKey Must not be null or empty - * @param secretKey Must not be null or empty - */ - public BasicAWSCredentials(String accessKey, String secretKey) { - Assert.hasText(accessKey, "The accessKey parameter must not be null or empty."); - Assert.hasText(secretKey, "The secretKey parameter must not be null or empty."); - - this.accessKey = accessKey; - this.secretKey = secretKey; - } - - /** - * Get the Access key to the Amazon WS account - */ - public String getAccessKey() { - return accessKey; - } - - /** - * Set the Access key to the Amazon WS account - */ - public void setAccessKey(String accessKey) { - Assert.hasText(accessKey, "The accessKey parameter must not be null or empty."); - this.accessKey = accessKey; - } - - /** - * Get the Secret key to the Amazon WS account - */ - public String getSecretKey() { - return secretKey; - } - - /**. - * Set the Secret key to the Amazon WS account - * @param secretKey - */ - public void setSecretKey(String secretKey) { - Assert.hasText(secretKey, "The secretKey parameter must not be null or empty."); - this.secretKey = secretKey; - } -} diff --git a/src/main/java/org/springframework/integration/aws/core/InvalidAWSCredentialsException.java b/src/main/java/org/springframework/integration/aws/core/InvalidAWSCredentialsException.java deleted file mode 100644 index 5abde9f..0000000 --- a/src/main/java/org/springframework/integration/aws/core/InvalidAWSCredentialsException.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.aws.core; - -/** - * Thrown when AWS Credentials provided by the user are incomplete or invalid - * - * @author Amol Nayak - * - * @since 0.5 - * - */ -public class InvalidAWSCredentialsException extends RuntimeException { - - /** - * - */ - private static final long serialVersionUID = 1L; - - public InvalidAWSCredentialsException() { - super(); - } - - public InvalidAWSCredentialsException(String message, Throwable cause) { - super(message, cause); - } - - public InvalidAWSCredentialsException(String message) { - super(message); - } - - public InvalidAWSCredentialsException(Throwable cause) { - super(cause); - } -} diff --git a/src/main/java/org/springframework/integration/aws/core/PropertiesAWSCredentials.java b/src/main/java/org/springframework/integration/aws/core/PropertiesAWSCredentials.java deleted file mode 100644 index ca14485..0000000 --- a/src/main/java/org/springframework/integration/aws/core/PropertiesAWSCredentials.java +++ /dev/null @@ -1,149 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.aws.core; - -import java.io.IOException; -import java.util.Properties; - -import org.springframework.beans.factory.InitializingBean; -import org.springframework.core.io.ClassPathResource; -import org.springframework.core.io.Resource; -import org.springframework.util.Assert; -import org.springframework.util.ClassUtils; -import org.springframework.util.ResourceUtils; -import org.springframework.util.StringUtils; - -/** - * Load the AWS credentials from the .properties file - * - * @author Amol Nayak - * - * @since 0.5 - * - */ -public class PropertiesAWSCredentials extends BasicAWSCredentials implements InitializingBean { - - - public static final String DEFAULT_AWS_ACCESS_KEY_PROPERTY = "accessKey"; - public static final String DEFAULT_AWS_SECRET_KEY_PROPERTY = "secretKey"; - - private String accessKeyProperty = DEFAULT_AWS_ACCESS_KEY_PROPERTY; - private String secretKeyProperty = DEFAULT_AWS_SECRET_KEY_PROPERTY; - private String propertyFileName; - - /**. - * Constructor accepting the properties file. - * - * @param propertyFileName Must not be null or empty - */ - public PropertiesAWSCredentials(String propertyFileName) { - super(); - Assert.hasText(propertyFileName, "The propertyFileName parameter must not be null or empty."); - this.propertyFileName = propertyFileName; - } - - /**. - * Gets the property name which holds the AWS access key - * @return the Access Key Property - */ - public String getAccessKeyProperty() { - return accessKeyProperty; - } - - /**. - * Sets the name of the property that will be used as the key in the properties - * file to hold the AWS access key - * @param accessKeyProperty - */ - public void setAccessKeyProperty(String accessKeyProperty) { - this.accessKeyProperty = accessKeyProperty; - } - - /**. - * Gets the property name which holds the - */ - public String getSecretKeyProperty() { - return secretKeyProperty; - } - - /**. - * Sets the name of the property that will be used as the key in the properties - * file to hold the AWS secret key - */ - public void setSecretKeyProperty(String secretKeyProperty) { - this.secretKeyProperty = secretKeyProperty; - } - - /**. - * Get the name of the property file that will hold the AWS credentials - */ - public String getPropertyFileName() { - return propertyFileName; - } - - /**. - * Sets the name of the file that will hold the AW credentials. - * - * @param propertyFileName Must not be null or empty - */ - public void setPropertyFileName(String propertyFileName) { - Assert.hasText(propertyFileName, "The propertyFileName parameter must not be null or empty."); - this.propertyFileName = propertyFileName; - } - - /**. - * Load the properties file and the keys - */ - public void afterPropertiesSet() throws Exception { - if (!StringUtils.hasText(propertyFileName)) - throw new InvalidAWSCredentialsException("Mandatory property propertyFileName expected"); - - if(!StringUtils.hasText(accessKeyProperty)) - throw new InvalidAWSCredentialsException("accessKeyValue has to be non empty and non null"); - - if(!StringUtils.hasText(secretKeyProperty)) - throw new InvalidAWSCredentialsException("secretKeyValue has to be non empty and non null"); - - loadProperties(); - - } - - /**. - * The private method that loads the properties from the .properties file and sets the access keys - */ - private void loadProperties() { - Resource resource; - if(propertyFileName.startsWith(ResourceUtils.CLASSPATH_URL_PREFIX)) { - resource = new ClassPathResource(propertyFileName.substring(ResourceUtils.CLASSPATH_URL_PREFIX.length()), ClassUtils.getDefaultClassLoader()); - } else { - resource = new ClassPathResource(propertyFileName, ClassUtils.getDefaultClassLoader()); - } - if(!resource.exists()) - throw new InvalidAWSCredentialsException("Unable to find resource \"" + propertyFileName + "\" in classpath"); - - Properties props = new Properties(); - try { - props.load(resource.getInputStream()); - } catch (IOException e) { - throw new InvalidAWSCredentialsException("Unable to load properties from \"" + propertyFileName + "\" in classpath"); - } - - setAccessKey((String)props.get(accessKeyProperty)); - setSecretKey((String)props.get(secretKeyProperty)); - - - } -} diff --git a/src/main/java/org/springframework/integration/aws/inbound/S3InboundFileSynchronizer.java b/src/main/java/org/springframework/integration/aws/inbound/S3InboundFileSynchronizer.java new file mode 100644 index 0000000..e6d02f9 --- /dev/null +++ b/src/main/java/org/springframework/integration/aws/inbound/S3InboundFileSynchronizer.java @@ -0,0 +1,67 @@ +/* + * Copyright 2016 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.aws.inbound; + +import org.springframework.expression.common.LiteralExpression; +import org.springframework.integration.aws.support.S3SessionFactory; +import org.springframework.integration.file.remote.session.Session; +import org.springframework.integration.file.remote.session.SessionFactory; +import org.springframework.integration.file.remote.synchronizer.AbstractInboundFileSynchronizer; + +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.S3ObjectSummary; + +/** + * An implementation of {@link AbstractInboundFileSynchronizer} for Amazon S3. + * + * @author Artem Bilan + */ +public class S3InboundFileSynchronizer extends AbstractInboundFileSynchronizer { + + public S3InboundFileSynchronizer() { + this(new S3SessionFactory()); + } + + public S3InboundFileSynchronizer(AmazonS3 amazonS3) { + this(new S3SessionFactory(amazonS3)); + } + + /** + * Create a synchronizer with the {@link SessionFactory} used to acquire {@link Session} instances. + * @param sessionFactory The session factory. + */ + public S3InboundFileSynchronizer(SessionFactory sessionFactory) { + super(sessionFactory); + setRemoteDirectoryExpression(new LiteralExpression(null)); + } + + @Override + protected boolean isFile(S3ObjectSummary file) { + return true; + } + + @Override + protected String getFilename(S3ObjectSummary file) { + return (file != null ? file.getKey() : null); + } + + @Override + protected long getModified(S3ObjectSummary file) { + return file.getLastModified().getTime(); + } + +} diff --git a/src/main/java/org/springframework/integration/aws/inbound/S3InboundFileSynchronizingMessageSource.java b/src/main/java/org/springframework/integration/aws/inbound/S3InboundFileSynchronizingMessageSource.java new file mode 100644 index 0000000..4d6f638 --- /dev/null +++ b/src/main/java/org/springframework/integration/aws/inbound/S3InboundFileSynchronizingMessageSource.java @@ -0,0 +1,49 @@ +/* + * Copyright 2016 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.aws.inbound; + +import java.io.File; +import java.util.Comparator; + +import org.springframework.integration.file.remote.synchronizer.AbstractInboundFileSynchronizer; +import org.springframework.integration.file.remote.synchronizer.AbstractInboundFileSynchronizingMessageSource; + +import com.amazonaws.services.s3.model.S3ObjectSummary; + +/** + * A {@link org.springframework.integration.core.MessageSource} implementation for the Amazon S3. + * + * @author Artem Bilan + */ +public class S3InboundFileSynchronizingMessageSource + extends AbstractInboundFileSynchronizingMessageSource { + + public S3InboundFileSynchronizingMessageSource(AbstractInboundFileSynchronizer synchronizer) { + super(synchronizer); + } + + public S3InboundFileSynchronizingMessageSource(AbstractInboundFileSynchronizer synchronizer, + Comparator comparator) { + super(synchronizer, comparator); + } + + + public String getComponentType() { + return "aws:s3-inbound-channel-adapter"; + } + +} diff --git a/src/main/java/org/springframework/integration/aws/inbound/SnsInboundChannelAdapter.java b/src/main/java/org/springframework/integration/aws/inbound/SnsInboundChannelAdapter.java index 990585f..43ce408 100644 --- a/src/main/java/org/springframework/integration/aws/inbound/SnsInboundChannelAdapter.java +++ b/src/main/java/org/springframework/integration/aws/inbound/SnsInboundChannelAdapter.java @@ -108,6 +108,11 @@ public class SnsInboundChannelAdapter extends HttpRequestHandlingMessagingGatewa } } + @Override + public String getComponentType() { + return "aws:sns-inbound-channel-adapter"; + } + @Override @SuppressWarnings("unchecked") protected void send(Object object) { diff --git a/src/main/java/org/springframework/integration/aws/inbound/SqsMessageDrivenChannelAdapter.java b/src/main/java/org/springframework/integration/aws/inbound/SqsMessageDrivenChannelAdapter.java index b55bbba..edec6ad 100644 --- a/src/main/java/org/springframework/integration/aws/inbound/SqsMessageDrivenChannelAdapter.java +++ b/src/main/java/org/springframework/integration/aws/inbound/SqsMessageDrivenChannelAdapter.java @@ -110,6 +110,11 @@ public class SqsMessageDrivenChannelAdapter extends MessageProducerSupport } } + @Override + public String getComponentType() { + return "aws:sqs-message-driven-channel-adapter"; + } + @Override protected void doStart() { this.listenerContainer.start(); diff --git a/src/main/java/org/springframework/integration/aws/s3/AbstractFileNameFilter.java b/src/main/java/org/springframework/integration/aws/s3/AbstractFileNameFilter.java deleted file mode 100644 index 6cb3174..0000000 --- a/src/main/java/org/springframework/integration/aws/s3/AbstractFileNameFilter.java +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.aws.s3; - - -import org.springframework.util.StringUtils; - -/** - * The abstract file name filter that first filters out the file if it is - * not eligible for filtering based on the name. - * For e.g, if a particular folder on S3 is to be synchronized with the - * local file system, then the name of the key is initially accepted - * only if it corresponds to an object under that folder or sub folder on S3. - * All other keys are ignored - * - * @author Amol Nayak - * - * @since 0.5 - * - */ -public abstract class AbstractFileNameFilter implements FileNameFilter { - - private volatile String folderName; - - private volatile boolean acceptSubFolders; - - /* (non-Javadoc) - * @see org.springframework.integration.aws.s3.FileNameFilter#accept(java.lang.String) - */ - public boolean accept(String fileName) { - if(!StringUtils.hasText(fileName)) - return false; - - if(StringUtils.hasText(folderName)) { - if(fileName.startsWith(folderName)) { - //This file is in the folder or in a child folder or the given folder - String relativePath = fileName.substring(folderName.length()); - if(relativePath.length() == 0 || (!acceptSubFolders && relativePath.indexOf("/") != -1)) { - return false; - } - } - else { - return false; - } - } - else { - //Its the folder entry within the bucket - if(!acceptSubFolders && fileName.indexOf("/") != -1) { - return false; - } - } - if(fileName.contains("/")) { - return isFileNameAccepted(fileName.substring(fileName.lastIndexOf("/") + 1)); - } - else { - return isFileNameAccepted(fileName); - } - - } - - - /** - * Gets the folder whose file are to be accepted, this path is relative to the - * bucket. - */ - public String getFolderName() { - return folderName; - } - - /** - * Sets the base folder name under which which the files will be accepted. - * - * @param folderName - */ - public void setFolderName(String folderName) { - - if(StringUtils.hasText(folderName)) { - String trimmedFolderName = folderName.trim(); - if("/".equals(trimmedFolderName)) { - trimmedFolderName = null; - } - else { - if(!trimmedFolderName.endsWith("/")) { - trimmedFolderName = trimmedFolderName + "/"; - } - - if(trimmedFolderName.startsWith("/")) { - trimmedFolderName = trimmedFolderName.substring(1); - } - } - this.folderName = trimmedFolderName; - } - else { - this.folderName = null; - } - - } - - /** - * Checks the flag if the sub folders are to be accepted or not. - * - */ - public boolean isAcceptSubFolders() { - return acceptSubFolders; - } - - - /** - * Sets if the sub folders of the folder set in {@link #folderName} - * are to be accepted or not. - * - * @param acceptSubFolders - */ - public void setAcceptSubFolders(boolean acceptSubFolders) { - this.acceptSubFolders = acceptSubFolders; - } - - - public abstract boolean isFileNameAccepted(String fileName); - -} diff --git a/src/main/java/org/springframework/integration/aws/s3/AlwaysTrueFileNamefilter.java b/src/main/java/org/springframework/integration/aws/s3/AlwaysTrueFileNamefilter.java deleted file mode 100644 index cd30af8..0000000 --- a/src/main/java/org/springframework/integration/aws/s3/AlwaysTrueFileNamefilter.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.aws.s3; - -/** - * Simple {@link FileNameFilter} implementation that accepts all the names. - * - * @author Amol Nayak - * - * @since 0.5 - * - */ -public class AlwaysTrueFileNamefilter extends AbstractFileNameFilter { - - /* (non-Javadoc) - * @see org.springframework.integration.aws.s3.FileNameFilter#accept(java.lang.String) - */ - - @Override - public boolean isFileNameAccepted(String fileName) { - return true; - } - -} diff --git a/src/main/java/org/springframework/integration/aws/s3/AmazonS3InboundSynchronizationMessageSource.java b/src/main/java/org/springframework/integration/aws/s3/AmazonS3InboundSynchronizationMessageSource.java deleted file mode 100644 index 9d9f2ae..0000000 --- a/src/main/java/org/springframework/integration/aws/s3/AmazonS3InboundSynchronizationMessageSource.java +++ /dev/null @@ -1,285 +0,0 @@ -/* - * Copyright 2002-2014 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.aws.s3; - -import java.io.File; -import java.util.concurrent.ArrayBlockingQueue; -import java.util.concurrent.BlockingQueue; - -import org.springframework.beans.factory.BeanFactory; -import org.springframework.context.expression.BeanFactoryResolver; -import org.springframework.expression.Expression; -import org.springframework.expression.spel.support.StandardEvaluationContext; -import org.springframework.messaging.Message; -import org.springframework.integration.aws.core.AWSCredentials; -import org.springframework.integration.aws.s3.core.AbstractAmazonS3Operations; -import org.springframework.integration.aws.s3.core.AmazonS3Operations; -import org.springframework.integration.aws.s3.core.DefaultAmazonS3Operations; -import org.springframework.integration.context.IntegrationObjectSupport; -import org.springframework.integration.core.MessageSource; -import org.springframework.integration.support.MessageBuilder; -import org.springframework.util.Assert; -import org.springframework.util.StringUtils; - -/** - * The message source used to receive the File instances stored on the local file system - * synchronized from the S3 - * - * @author Amol Nayak - * @author Rob Harrop - * - * @since 0.5 - * - */ -public class AmazonS3InboundSynchronizationMessageSource extends - IntegrationObjectSupport implements MessageSource,FileEventHandler { - - - private volatile InboundFileSynchronizer synchronizer; - private volatile String bucket; - private volatile String remoteDirectory; - private volatile File directory; - private volatile AWSCredentials credentials; - private volatile String temporarySuffix = ".writing"; - private volatile int maxObjectsPerBatch; - private volatile String fileNameWildcard; - private volatile String fileNameRegex; - private volatile BlockingQueue filesQueue; - private volatile AmazonS3Operations s3Operations; - private volatile boolean acceptSubFolders; - private volatile String awsEndpoint; - //We will hard code the queue capacity here - private final int QUEUE_CAPACITY = 1024; - private volatile StandardEvaluationContext ctx; - private volatile Expression directoryExpression; - - - public Message receive() { - File headElement = filesQueue.poll(); - if(headElement == null) { - synchronizer.synchronizeToLocalDirectory(directory, bucket, remoteDirectory); - //Now check the queue again - headElement = filesQueue.poll(); - } - if(headElement != null) { - return MessageBuilder.withPayload(headElement).build(); - } - else { - return null; - } - } - - - @Override - protected void onInit() throws Exception { - Assert.notNull(directoryExpression, "Local directory to synchronize to is not set"); - - ctx = new StandardEvaluationContext(); - BeanFactory factory = getBeanFactory(); - if(factory != null) { - ctx.setBeanResolver(new BeanFactoryResolver(factory)); - } - String directoryPath = directoryExpression.getValue(ctx,String.class); - directory = new File(directoryPath); - - Assert.notNull(directory, "Please provide a valid local directory to synchronize the remote files"); -// TODO: Uncomment this once we start supporting auto-create-local-directory -// Assert.isTrue(directory.exists(), -// String.format("Provided directory %s does not exist", directoryPath)); - Assert.isTrue(directory.isDirectory(), - String.format("Provided path %s is not a directory", directoryPath)); - - //instantiate the S3Operations instance - if(s3Operations == null) { - s3Operations = new DefaultAmazonS3Operations(credentials); - } - - if(AbstractAmazonS3Operations.class.isAssignableFrom(s3Operations.getClass())) { - AbstractAmazonS3Operations abstractOperation = (AbstractAmazonS3Operations)s3Operations; - abstractOperation.setTemporaryFileSuffix(temporarySuffix); - if(StringUtils.hasText(awsEndpoint)) { - abstractOperation.setAwsEndpoint(awsEndpoint); - } - abstractOperation.afterPropertiesSet(); - } - - //Now the file operations class - InboundLocalFileOperationsImpl fileOperations = new InboundLocalFileOperationsImpl(); - fileOperations.setTemporaryFileSuffix(temporarySuffix); - fileOperations.addEventListener(this); - - InboundFileSynchronizationImpl synchronizationImpl = - new InboundFileSynchronizationImpl(s3Operations, fileOperations); - synchronizationImpl.setSynchronizingBatchSize(maxObjectsPerBatch); - if(StringUtils.hasText(fileNameWildcard)) { - synchronizationImpl.setFileWildcard(fileNameWildcard); - } - if(StringUtils.hasText(fileNameRegex)) { - synchronizationImpl.setFileNamePattern(fileNameRegex); - } - synchronizationImpl.setAcceptSubFolders(acceptSubFolders); - synchronizationImpl.afterPropertiesSet(); - this.synchronizer = synchronizationImpl; - - filesQueue = new ArrayBlockingQueue(QUEUE_CAPACITY); - } - - //-- For Spring DI - - /** - * Sets the AWSCredential instance to be used - */ - public void setCredentials(AWSCredentials credentials) { - Assert.notNull(credentials, "null 'credentials' provided"); - this.credentials = credentials; - } - - - /** - * The temporary suffix that would be used to indicate that the file is being writtem and the operation - * is not yet complete - * - * @param temporarySuffix - */ - public void setTemporarySuffix(String temporarySuffix) { - Assert.hasText(temporarySuffix,"Provide a non null non empty string as temporary suffix"); - this.temporarySuffix = temporarySuffix; - } - - /** - * The maximum number of objects those will be retrieved in one batch from Amazon S3 bucket - * as part of the listOperation - * - * @param maxObjectsPerBatch - */ - public void setMaxObjectsPerBatch(int maxObjectsPerBatch) { - Assert.isTrue(maxObjectsPerBatch > 0, "Provide a non sero, non negative number for max objects per batch"); - this.maxObjectsPerBatch = maxObjectsPerBatch; - } - - /** - * Sets the file's wildcard pattern that would be used to match the objects in S3 bucket - * This attribute is mutually exclusive to fileName regex. - * - * @param fileNameWildcard Must not be empty. - */ - public void setFileNameWildcard(String fileNameWildcard) { - Assert.hasText(fileNameWildcard, "Provided file wildcard is null or empty string"); - Assert.isTrue(!StringUtils.hasText(fileNameRegex), "File name regex and wildcard are mutually exclusive"); - this.fileNameWildcard = fileNameWildcard; - } - - /** - * Sets the regex to be used to match the objects in S3 bucket. This attribute is mutually exclusive - * to fileName regex. - * - * @param fileNameRegex - */ - public void setFileNameRegex(String fileNameRegex) { - Assert.hasText(fileNameRegex, "Provided file regex is null or empty string"); - Assert.isTrue(!StringUtils.hasText(fileNameWildcard), "File name regex and wildcard are mutually exclusive"); - this.fileNameRegex = fileNameRegex; - } - - /** - * Sets the bucket with which the data in local directory is synchronized with. - * - * @param bucket - */ - public void setBucket(String bucket) { - Assert.hasText(bucket, "Provided 'bucket' is null or empty string"); - this.bucket = bucket; - } - - /** - * Sets the remote directory, this is the directory relative to the provided bucket - * in S3. - * - * @param remoteDirectory - */ - public void setRemoteDirectory(String remoteDirectory) { - Assert.hasText(remoteDirectory, "Provided 'remoteDirectory' is null or empty string"); - this.remoteDirectory = remoteDirectory; - } - - /** - * Sets the expression to find the local directory where the remote files are synchronized with. - * - * @param directoryExpression Must not be null - */ - public void setDirectory(Expression directoryExpression) { - Assert.notNull(directoryExpression, "provided 'directoryExpression' is null"); - this.directoryExpression = directoryExpression; - } - - /** - * Sets the {@link AmazonS3Operations} instance that would be used for the receiving - * the objects and listing the objects in the bucket. - * - * @param s3Operations - */ - public void setS3Operations(AmazonS3Operations s3Operations) { - Assert.notNull(s3Operations, "null 's3Operations' instance provided"); - this.s3Operations = s3Operations; - } - - /** - * Set to true if you want the subfolders of the given remote folder to be synchronized to the - * local directory. - * - * @param acceptSubFolders - */ - public void setAcceptSubFolders(boolean acceptSubFolders) { - this.acceptSubFolders = acceptSubFolders; - } - - /** - * The AWS region's endpoint whose bucket(and the subfolder if any) will be synchronized - * by this adapter - * - * @param awsEndpoint - */ - public void setAwsEndpoint(String awsEndpoint) { - Assert.hasText(awsEndpoint, "Given AWS Endpoint has to be non null and non empty string"); - this.awsEndpoint = awsEndpoint; - } - - - //---- - - - - - - public void onEvent(FileEvent event) { - //We are interested in Create new file events only - if(FileOperationType.CREATE.equals(event.getFileOperation())) { - try { - filesQueue.put(event.getFile()); - //The call hierarchy is - //if, no file found in queue, then - // receive() - // -> InboundFileSynchronizer.synchronizeToLocalDirectory() - // ->InboundLocalFileOperations.writeToFile() - // ->onEvent() - //If the Queue is full and the thread blocks, the lock in synchronizeToLocalDirectory - //stays and hence preventing further concurrent synchronization - } catch (InterruptedException e) { - logger.error("Interrupted while waiting to put the event on the filesQueue", e); - } - } - } -} diff --git a/src/main/java/org/springframework/integration/aws/s3/AmazonS3MessageHeaders.java b/src/main/java/org/springframework/integration/aws/s3/AmazonS3MessageHeaders.java deleted file mode 100644 index cac5082..0000000 --- a/src/main/java/org/springframework/integration/aws/s3/AmazonS3MessageHeaders.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.aws.s3; - -/** - * Constants defining the headers containing attributes of the S3 object like - * File Name, User's metadata, Object metadata, Object ACL etc - * - * @author Amol Nayak - * - * @since 0.5 - * - */ -public interface AmazonS3MessageHeaders { - - //TODO: Get rid of the interface for constants - public static final String FILE_NAME = "file_name"; - public static final String USER_METADATA = "user_meta_data"; - public static final String METADATA = "meta_data"; - public static final String OBJECT_ACLS = "object_acls"; -} diff --git a/src/main/java/org/springframework/integration/aws/s3/AmazonS3ObjectBuilder.java b/src/main/java/org/springframework/integration/aws/s3/AmazonS3ObjectBuilder.java deleted file mode 100644 index b6208ad..0000000 --- a/src/main/java/org/springframework/integration/aws/s3/AmazonS3ObjectBuilder.java +++ /dev/null @@ -1,207 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.aws.s3; - -import java.io.File; -import java.io.InputStream; -import java.util.Collection; -import java.util.Map; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.springframework.integration.aws.s3.core.AmazonS3Object; -import org.springframework.integration.aws.s3.core.AmazonS3ObjectACL; -import org.springframework.integration.aws.s3.core.Grantee; -import org.springframework.integration.aws.s3.core.GranteeType; -import org.springframework.integration.aws.s3.core.ObjectGrant; -import org.springframework.integration.aws.s3.core.ObjectPermissions; -import org.springframework.util.Assert; -import org.springframework.util.StringUtils; - -/** - * The convenience builder class for building {@link AmazonS3Object} - * - * @author Amol Nayak - * - * @since 0.5 - * - */ -public class AmazonS3ObjectBuilder { - - private final Log logger = LogFactory.getLog(getClass()); - - private File file; - private InputStream in; - private Map metaData; - private Map userMetaData; - private AmazonS3ObjectACL objectACL; - - /** - * Gets a new instance of the builder - */ - public static AmazonS3ObjectBuilder getInstance() { - return new AmazonS3ObjectBuilder(); - } - - /** - * Sets the file which is to be read for uploading into S3 - * @param file - */ - public AmazonS3ObjectBuilder fromFile(File file) { - Assert.notNull(file,"null 'file' object provided"); - Assert.isNull(in, "Cannot instantiate using both the InputStream and File"); - Assert.isTrue(file.exists(), "Provided File location \""+ file.getAbsolutePath() - + "\" is invalid"); - Assert.isTrue(!file.isDirectory(), "Provided File location \""+ file.getAbsolutePath() - + "\" is a directory path, please provide a valid file location"); - this.file = file; - return this; - } - - /** - * Convenience method for setting the File object from the String path - * @param fileLocation - */ - public AmazonS3ObjectBuilder fromLocation(String fileLocation) { - Assert.hasText(fileLocation, "'fileLocation should be non null, non empty string"); - return fromFile(new File(fileLocation)); - } - - /** - * Sets an InputStream from which the data to be uploaded to S3 will be read - * @param in The provided {@link InputStream} must not be null - */ - public AmazonS3ObjectBuilder fromInputStream(InputStream in) { - Assert.notNull(in, "The Stream object provided is null"); - Assert.isNull(file, "Cannot instantiate using both the InputStream and File"); - this.in = in; - return this; - } - - /** - * Use the given user meta data for the file to be uploaded - * @param userMetaData - */ - public AmazonS3ObjectBuilder withUserMetaData(Map userMetaData) { - this.userMetaData = userMetaData; - return this; - } - - /** - * uses the given metadata for the S3 object to be uploaded - * @param metaData - */ - public AmazonS3ObjectBuilder withMetaData(Map metaData) { - this.metaData = metaData; - return this; - } - - /** - * Sets the S3 Object ACL - * @param objectACL - */ - public AmazonS3ObjectBuilder withObjectACL(Map> objectACL) { - //The key can be of three types, the email id, the canonical id of the user or the Group identifier - if(objectACL != null && !objectACL.isEmpty()) { - this.objectACL = new AmazonS3ObjectACL(); - for(String key:objectACL.keySet()) { - if(isCanonicalId(key)) { - addPermissions(key, GranteeType.CANONICAL_GRANTEE_TYPE, objectACL.get(key)); - } - else if(isGroupIdentifier(key)) { - addPermissions(key, GranteeType.GROUP_GRANTEE_TYPE, objectACL.get(key)); - } - else { - //assuming thats an email identifier, not using regex to validate the email - //id. We can add email validation if needed and throw an eexception - //if some unexpected value comes up. - addPermissions(key, GranteeType.EMAIL_GRANTEE_TYPE, objectACL.get(key)); - } - } - } - return this; - } - - /** - * Internal helper method for adding the Grants to the object ACL - * @param grantee - * @param type - * @param permissions - */ - private void addPermissions(String grantee,GranteeType type,Collection permissions) { - for(String permission:permissions) { - ObjectPermissions objectPermission = getObjectPermission(permission); - this.objectACL.addGrant(new ObjectGrant - (new Grantee(grantee,type), objectPermission)); - } - } - - /** - * Gets the Appropriate {@link ObjectPermissions} based on the String passed - * @param permission - */ - private ObjectPermissions getObjectPermission(String permission) { - //Types of permission are READ, READ_ACP, WRITE_ACP - if(StringUtils.hasText(permission)) { - String permissionString = permission.trim().replaceAll("[ ]+", "_").toUpperCase(); - try { - return ObjectPermissions.valueOf(permissionString); - } catch (IllegalArgumentException e) { - logger.error("Requested Enum not found, see underlying exception for more details", e); - throw e; - } - } - else { - throw new IllegalArgumentException("Empty or null string found for object permission"); - } - } - - /** - * Method checks if the provided String is a canonical id of the AWS account - * @param identifier - */ - private boolean isCanonicalId(String identifier) { - //Note: we do not check if the given id is a valid canonical id with AWS, we just check if it is in the right format - if(StringUtils.hasText(identifier)) { - if(identifier.length() == 64) { - String replacedString = identifier.trim().replaceAll("[a-fA-F0-9]+",""); - return replacedString.length() == 0; - } - } - return false; - } - - /** - * Checks if the given identifier corresponds to a group id - * @param identifier - */ - private boolean isGroupIdentifier(String identifier){ - if(StringUtils.hasText(identifier)) { - String trimmedIdentifier = identifier.trim(); - return trimmedIdentifier.equals("http://acs.amazonaws.com/groups/global/AllUsers") - || trimmedIdentifier.equals("http://acs.amazonaws.com/groups/global/AuthenticatedUsers"); - } - return false; - } - - /** - * Builds the {@link AmazonS3Object} from the provided meta data, file/input stream and object ACLs - */ - public AmazonS3Object build() { - Assert.isTrue(!(in == null && file == null),"One of File object or InputStream is required"); - return new AmazonS3Object(userMetaData, metaData, in, file,objectACL); - } -} diff --git a/src/main/java/org/springframework/integration/aws/s3/DefaultFileNameGenerationStrategy.java b/src/main/java/org/springframework/integration/aws/s3/DefaultFileNameGenerationStrategy.java deleted file mode 100644 index 919aff4..0000000 --- a/src/main/java/org/springframework/integration/aws/s3/DefaultFileNameGenerationStrategy.java +++ /dev/null @@ -1,101 +0,0 @@ -/* - * Copyright 2002-2014 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.aws.s3; - -import java.io.File; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.springframework.messaging.Message; -import org.springframework.integration.util.AbstractExpressionEvaluator; -import org.springframework.util.Assert; -import org.springframework.util.StringUtils; - -/** - * The Default file name generation strategy. The strategy does the below steps for file name - * generation - * 1. The expression provided for generation of the file name, it is evaluated and - * if a value is obtained, it is used. By default it used the value present in the - * file_name header. - * 2. Else, if the provided payload is of type {@link File}, then the name of the file is used. - * if the file name ends with the temporary file suffix, the suffix is removed and the - * remainder of the file is used as the name. - * 3. If none of the above two are provided, the file name is the <Message Id>.ext - * - * @author Amol Nayak - * @author Rob Harrop - * - * @since 0.5 - * - */ -public class DefaultFileNameGenerationStrategy extends AbstractExpressionEvaluator implements - FileNameGenerationStrategy { - - private final Log logger = LogFactory.getLog(DefaultFileNameGenerationStrategy.class); - - private volatile String temporarySuffix = ".writing"; - - private volatile String fileNameExpression = "headers['" + AmazonS3MessageHeaders.FILE_NAME + "']" ; - - /* (non-Javadoc) - * @see org.springframework.integration.aws.s3.FileNameGenerationStrategy#generateFileName(org.springframework.messaging.Message) - */ - public String generateFileName(Message message) { - String generatedFileName; - try { - String fileName = evaluateExpression(fileNameExpression, message,String.class); - if(StringUtils.hasText(fileName)) { - return fileName; - } - } catch (Exception e) { - //Some exception while evaluating using expression, continue to the file Name - //Ignore - logger.warn("Exception while evaluating the expression '" - + fileNameExpression + "' on the message", e); - } - Object payload = message.getPayload(); - if(payload instanceof File) { - String fileName = ((File)payload).getName(); - if(fileName.endsWith(temporarySuffix)) { - //chop off the temp suffix - generatedFileName = fileName.substring(0, fileName.indexOf(temporarySuffix)); - } - else { - generatedFileName = fileName; - } - } - else { - //use the default name generated - generatedFileName = message.getHeaders().getId() + ".ext"; - } - if(logger.isInfoEnabled()) { - logger.info("Generated file name is " + generatedFileName); - } - - return generatedFileName; - - } - - public void setTemporarySuffix(String temporarySuffix) { - Assert.hasText(temporarySuffix, "Temporary directory suffix should be non null, non empty"); - this.temporarySuffix = temporarySuffix; - } - - public void setFileNameExpression(String fileNameExpression) { - Assert.hasText(fileNameExpression, "Remote name generation expression should be non null, non empty string"); - this.fileNameExpression = fileNameExpression; - } -} diff --git a/src/main/java/org/springframework/integration/aws/s3/FileEvent.java b/src/main/java/org/springframework/integration/aws/s3/FileEvent.java deleted file mode 100644 index e97d611..0000000 --- a/src/main/java/org/springframework/integration/aws/s3/FileEvent.java +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.aws.s3; - -import java.io.File; - - -/** - * The interface denoting the file event type and the {@link File} on which the event occurred - * - * @author Amol Nayak - * - * @since 0.5 - * - */ -public interface FileEvent { - - /** - * The Type of the operation on the {@link File} that occurred - * @return The {@link FileOperationType} - */ - FileOperationType getFileOperation(); - - /** - * The File on which the given occurred - */ - File getFile(); -} diff --git a/src/main/java/org/springframework/integration/aws/s3/FileEventHandler.java b/src/main/java/org/springframework/integration/aws/s3/FileEventHandler.java deleted file mode 100644 index eaad098..0000000 --- a/src/main/java/org/springframework/integration/aws/s3/FileEventHandler.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.aws.s3; - -/** - * Callback interface that gets notified upon a file event on local file system which are - * performed from {@link InboundLocalFileOperations} - * - * @author Amol Nayak - * - * @since 0.5 - * - */ -public interface FileEventHandler { - - /** - * The Method that would be invoked with the specified event that occurred on the file - * @param event the {@link FileEvent} - */ - void onEvent(FileEvent event); -} diff --git a/src/main/java/org/springframework/integration/aws/s3/FileNameFilter.java b/src/main/java/org/springframework/integration/aws/s3/FileNameFilter.java deleted file mode 100644 index 2fe529b..0000000 --- a/src/main/java/org/springframework/integration/aws/s3/FileNameFilter.java +++ /dev/null @@ -1,34 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.aws.s3; - -/** - * The strategy interface used to filter out file names based on some predetermined criteria - * - * @author Amol Nayak - * - * @since 0.5 - * - */ -public interface FileNameFilter { - - /** - * Determines whether to accept the file with the given name or not - * @param fileName - * @return true if the file with the given name can be accepted, else false - */ - boolean accept(String fileName); -} diff --git a/src/main/java/org/springframework/integration/aws/s3/FileNameGenerationStrategy.java b/src/main/java/org/springframework/integration/aws/s3/FileNameGenerationStrategy.java deleted file mode 100644 index 67caf91..0000000 --- a/src/main/java/org/springframework/integration/aws/s3/FileNameGenerationStrategy.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Copyright 2002-2014 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.aws.s3; - -import org.springframework.messaging.Message; - -/** - * The Common interface used to generate the file name for the file to be uploaded to S3 - * - * @author Amol Nayak - * @author Rob Harrop - * - * @since 0.5 - * - */ -public interface FileNameGenerationStrategy { - - /** - * Generates the file name from the given message - */ - String generateFileName(Message message); -} diff --git a/src/main/java/org/springframework/integration/aws/s3/FileOperationType.java b/src/main/java/org/springframework/integration/aws/s3/FileOperationType.java deleted file mode 100644 index 5340134..0000000 --- a/src/main/java/org/springframework/integration/aws/s3/FileOperationType.java +++ /dev/null @@ -1,28 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.aws.s3; - -/** - * The enum for various file operations - * - * @author Amol Nayak - * - * @since 0.5 - * - */ -public enum FileOperationType { - CREATE,UPDATE,RENAME,DELETE; -} diff --git a/src/main/java/org/springframework/integration/aws/s3/InboundFileSynchronizationImpl.java b/src/main/java/org/springframework/integration/aws/s3/InboundFileSynchronizationImpl.java deleted file mode 100644 index 106ade7..0000000 --- a/src/main/java/org/springframework/integration/aws/s3/InboundFileSynchronizationImpl.java +++ /dev/null @@ -1,349 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.aws.s3; - -import static org.springframework.integration.aws.core.AWSCommonUtils.decodeBase64; -import static org.springframework.integration.aws.core.AWSCommonUtils.encodeHex; -import static org.springframework.integration.aws.core.AWSCommonUtils.getContentsMD5AsBytes; - -import java.io.File; -import java.io.IOException; -import java.io.UnsupportedEncodingException; -import java.util.List; -import java.util.Map; -import java.util.concurrent.locks.Lock; -import java.util.concurrent.locks.ReentrantLock; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import org.springframework.beans.factory.InitializingBean; -import org.springframework.integration.aws.s3.core.AmazonS3Object; -import org.springframework.integration.aws.s3.core.AmazonS3Operations; -import org.springframework.integration.aws.s3.core.PaginatedObjectsView; -import org.springframework.integration.aws.s3.core.S3ObjectSummary; -import org.springframework.util.Assert; -import org.springframework.util.StringUtils; - - -/** - * The implementation for {@link InboundFileSynchronizer}, this implementation will use - * the {@link AmazonS3Operations} to list the objects in the remote bucket on invocation of - * the {@link #synchronizeToLocalDirectory(File, String, String)}. The listed objects will then - * be checked against the - * - * @author Amol Nayak - * @author Christos Kapasakalidis - * - * @since 0.5 - * - */ -public class InboundFileSynchronizationImpl implements InboundFileSynchronizer, InitializingBean { - - private final Log logger = LogFactory.getLog(getClass()); - - public static final String CONTENT_MD5 = "Content-MD5"; - - private final AmazonS3Operations client; - - private volatile int maxObjectsPerBatch = 100; //default - - private final InboundLocalFileOperations fileOperations; - - private volatile FileNameFilter filter; - - private volatile String fileWildcard; - - private volatile String fileNameRegex; - - private final Lock lock = new ReentrantLock(); - - private volatile boolean acceptSubFolders; - - /** - * - * @param client - */ - public InboundFileSynchronizationImpl(AmazonS3Operations client, - InboundLocalFileOperations fileOperations) { - Assert.notNull(client, "AmazonS3Client should be non null"); - Assert.notNull(fileOperations, "fileOperations should be non null"); - this.client = client; - this.fileOperations = fileOperations; - } - - - public void afterPropertiesSet() throws Exception { - Assert.isTrue(!(StringUtils.hasText(fileWildcard) && StringUtils.hasText(fileNameRegex)), - "Only one of the file name wildcard string or file name regex can be specified"); - - if (StringUtils.hasText(fileWildcard)) { - filter = new WildcardFileNameFilter(fileWildcard); - } - else if (StringUtils.hasText(fileNameRegex)) { - filter = new RegexFileNameFilter(fileNameRegex); - } - else { - filter = new AlwaysTrueFileNamefilter(); //Match all - } - - if (acceptSubFolders) { - ((AbstractFileNameFilter) filter).setAcceptSubFolders(true); - fileOperations.setCreateDirectoriesIfRequired(true); - } - } - - - /* (non-Javadoc) - * @see org.springframework.integration.aws.s3.InboundFileSynchronizer#synchronizeToLocalDirectory(java.io.File, java.lang.String, java.lang.String) - */ - - public void synchronizeToLocalDirectory(File localDirectory, String bucketName, String remoteFolder) { - if (!lock.tryLock()) { - if (logger.isInfoEnabled()) { - logger.info("Sync already in progess"); - } - //Prevent concurrent synchronization requests - return; - } - - if (logger.isInfoEnabled()) { - logger.info("Starting sync with local directory"); - } - //Below sync can take long, above lock ensures only one thread is synchronizing - try { - if (remoteFolder != null && "/".equals(remoteFolder)) { - remoteFolder = null; - } - - //Set the remote folder for the filter - if (filter instanceof AbstractFileNameFilter) { - ((AbstractFileNameFilter) filter).setFolderName(remoteFolder); - } - - String nextMarker = null; - do { - PaginatedObjectsView paginatedView = client.listObjects(bucketName, remoteFolder, nextMarker, maxObjectsPerBatch); - if (paginatedView == null) - break; //No files to sync - nextMarker = paginatedView.getNextMarker(); - List summaries = paginatedView.getObjectSummary(); - for (S3ObjectSummary summary : summaries) { - String key = summary.getKey(); - if (key.endsWith("/")) { - continue; - } - if (!filter.accept(key)) - continue; - //The folder is the root as the key is relative to bucket - AmazonS3Object s3Object = null; - try { - s3Object = client.getObject(bucketName, "/", key); - synchronizeObjectWithFile(localDirectory, summary, s3Object); - } - finally { - if (s3Object != null && s3Object.getInputStream() != null) { - s3Object.getInputStream().close(); - } - } - } - } while (nextMarker != null); - - } - catch (IOException e) { - logger.error("Caught Exception while trying to close s3object InputStream", e); - } - finally { - lock.unlock(); - if (logger.isInfoEnabled()) { - logger.info("Sync completed"); - } - } - } - - /** - * Synchronizes the Object with the File on the local file system - * @param localDirectory - * @param summary - */ - private void synchronizeObjectWithFile(File localDirectory, S3ObjectSummary summary, - AmazonS3Object s3Object) { - //Get the complete object data - - String key = summary.getKey(); - if (key.endsWith("/")) { - return; - } - int lastIndex = key.lastIndexOf("/"); - - String fileName = key.substring(lastIndex + 1); - String filePath = localDirectory.getAbsolutePath(); - if (!filePath.endsWith(File.separator)) { - filePath += File.separator; - } - - File baseDirectory; - if (lastIndex > 0) { - //there could very well be previous '/' and thus nested sub folders - String prefixKey = key.substring(0, lastIndex); - String[] folders = prefixKey.split("/"); - if (folders.length > 0) { - for (String folder : folders) { - filePath = filePath + folder + File.separator; - } - //create the directory structure - baseDirectory = new File(filePath); - baseDirectory.mkdirs(); - } - else { - baseDirectory = localDirectory; - } - } - else { - baseDirectory = localDirectory; - } - - File file = new File(filePath + fileName); - if (!file.exists()) { - //File doesnt exist, write the contents to it - try { - fileOperations.writeToFile(baseDirectory, fileName, s3Object.getInputStream()); - } - catch (IOException e) { - logger.error("Caught Exception while writing to file " + file.getAbsolutePath()); - //continue with next file. - } - } - else { - //Synchronize a file that exists - if (!file.isFile()) { - if (logger.isWarnEnabled()) { - logger.warn("The file " + file.getAbsolutePath() + " is not a regular file, probably a directory, "); - } - return; - } - String eTag = summary.getETag(); - String md5Hex = null; - if (isEtagMD5Hash(eTag)) { - //Single thread upload - try { - md5Hex = encodeHex(getContentsMD5AsBytes(file)); - } - catch (UnsupportedEncodingException e) { - logger.error("Exception encountered while generating the MD5 hash for the file " + file.getAbsolutePath(), e); - } - if (!eTag.equals(md5Hex)) { - //The local file is different than the one on S3, could be latest but we will still - //sync this with the copy on S3 - try { - fileOperations.writeToFile(baseDirectory, fileName, s3Object.getInputStream()); - } - catch (IOException e) { - logger.error("Caught Exception while writing to file " + file.getAbsolutePath()); - } - } - } - else { - //Multi part upload - //Get the MD5 hash from the headers - Map userMetaData = s3Object.getUserMetaData(); - String b64MD5 = userMetaData.get(CONTENT_MD5); - if (b64MD5 != null) { - //Need to convert to Hex from Base64 - try { - md5Hex = encodeHex(getContentsMD5AsBytes(file)); - } - catch (UnsupportedEncodingException e) { - logger.error("Exception encountered while generating the MD5 hash for the file " + file.getAbsolutePath(), e); - } - try { - String remoteHexMD5 = new String( - encodeHex( - decodeBase64(b64MD5.getBytes("UTF-8")))); - if (!md5Hex.equals(remoteHexMD5)) { - //Update only if the local file is not same as remote file - try { - fileOperations.writeToFile(baseDirectory, fileName, s3Object.getInputStream()); - } - catch (IOException e) { - logger.error("Caught Exception while writing to file " + file.getAbsolutePath()); - } - } - - } - catch (UnsupportedEncodingException e) { - //Should never get this, suppress - } - } - else { - //Forcefully update the file - try { - fileOperations.writeToFile(baseDirectory, fileName, s3Object.getInputStream()); - } - catch (IOException e) { - logger.error("Caught Exception while writing to file " + file.getAbsolutePath()); - } - } - } - } - } - - /** - * Checks if the given eTag is a MD5 hash as hex, the hash is 128 bit and hence - * has to be 32 characters in length, also it should contain only hex characters - * In case of multi uploads, it is observed that the eTag contains a "-", - * and hence this method will return false. - * - * @param eTag - */ - private boolean isEtagMD5Hash(String eTag) { - if (eTag == null || eTag.length() != 32) { - return false; - } - return eTag.replaceAll("[a-f0-9A-F]", "").length() == 0; - - } - - /* (non-Javadoc) - * @see org.springframework.integration.aws.s3.InboundFileSynchronizer#setSynchronizingBatchSize(int) - */ - - public void setSynchronizingBatchSize(int batchSize) { - if (batchSize > 0) - this.maxObjectsPerBatch = batchSize; - } - - /* (non-Javadoc) - * @see org.springframework.integration.aws.s3.InboundFileSynchronizer#setFileNamePattern(java.lang.String) - */ - - public void setFileNamePattern(String fileNameRegex) { - this.fileNameRegex = fileNameRegex; - } - - /* (non-Javadoc) - * @see org.springframework.integration.aws.s3.InboundFileSynchronizer#setFileWildcard(java.lang.String) - */ - - public void setFileWildcard(String fileWildcard) { - this.fileWildcard = fileWildcard; - } - - @Override - public void setAcceptSubFolders(boolean acceptSubFolders) { - this.acceptSubFolders = acceptSubFolders; - } -} diff --git a/src/main/java/org/springframework/integration/aws/s3/InboundFileSynchronizer.java b/src/main/java/org/springframework/integration/aws/s3/InboundFileSynchronizer.java deleted file mode 100644 index d38a975..0000000 --- a/src/main/java/org/springframework/integration/aws/s3/InboundFileSynchronizer.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.aws.s3; - -import java.io.File; - -/** - * The strategy interface for synchronizion the remote file system with local directory - * - * @author Amol Nayak - * - * @since 0.5 - * - */ -public interface InboundFileSynchronizer { - - /** - * The operation synchronizes the remote S3 file system with the local directory - * It retrieved new files and updates existing ones with the latest content from S3 - * Please note that this method will NOT delete any additional files present on the - * local filesystem - * @param localDirectory The local directory that needs to be synchronized with the Remote S3 bucket - * @param bucketName The name of the bucket whose contents are to be synchronized - * @param remoteFolder The folder name in S3 whose contents are to be synchronized - * use / if the contents of the bucket starting from the root - * are to be synchronized. This operation will only synchronize - * the files resent in the given remote folder and will ignore - * all the folders and sub folder in it. - */ - void synchronizeToLocalDirectory(File localDirectory,String bucketName,String remoteFolder); - - /** - * Sets the max number of Files to synchronize at a time. This is the max number of - * Object information retrieved at a time from S3 and synchronized with the Local file system - * before the next set of information is retrieved from S3. Please note if the file name - * pattern is set the total number of files from a batch fetched can be anywhere between - * 0 to the max number of files fetched per batch - * @param batchSize - */ - void setSynchronizingBatchSize(int batchSize); - - /** - * Sets the file name regex that will be used to match the key value from S3 - * to find a match. - * @param fileNameRegex - */ - void setFileNamePattern(String fileNameRegex); - - /** - * Sets the simple file name wildcard to match to match the file e.g., it can be - * set to *.txt to accept all .txt files - */ - void setFileWildcard(String wildcardString); - - /** - * Set the value to true to enable the objects to be accepted even if they - * are present in the sub folder of the remote folder passed to - * {@link #synchronizeToLocalDirectory(File, String, String)} - * - * @param acceptSubFolders - */ - void setAcceptSubFolders(boolean acceptSubFolders); -} diff --git a/src/main/java/org/springframework/integration/aws/s3/InboundLocalFileOperations.java b/src/main/java/org/springframework/integration/aws/s3/InboundLocalFileOperations.java deleted file mode 100644 index 890ecf1..0000000 --- a/src/main/java/org/springframework/integration/aws/s3/InboundLocalFileOperations.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright 2002-2014 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.aws.s3; - -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.util.List; - -/** - * Interface for performing File Operations on local file system. - * It supports registering an {@link FileEventHandler} instances that - * notifies the operations on the file those were performed - * - * @author Amol Nayak - * @author Rob Harrop - * - * @since 0.5 - * - */ -public interface InboundLocalFileOperations { - - /** - * Registers an individual event handler. - * @param handler - */ - void addEventListener(FileEventHandler handler); - - /** - * Registers a {@link List} of {@link FileEventHandler} instances - * @param handlers - */ - void setEventListeners(List handlers); - - /** - * The temporary file suffix that will be used when the file is being written to the filesystem - * @param prefix - */ - void setTemporaryFileSuffix(String prefix); - - - /** - * Sets the flag to true if directories given are to be created if not present - * - * @param createDirectoriesIfRequired - */ - void setCreateDirectoriesIfRequired(boolean createDirectoriesIfRequired); - - /** - * The method will write to the file with the specified name in the specified directory - * from the given {@link InputStream}. Upon completion of the writing the appropriate - * {@link FileEventHandler} instance(s) will be notified with the {@link FileOperationType} - * WRITE and {@link File} instance for the created file. - * @param directory - * @param fileName - * @param in - */ - void writeToFile(File directory,String fileName,InputStream in) throws IOException ; -} diff --git a/src/main/java/org/springframework/integration/aws/s3/InboundLocalFileOperationsImpl.java b/src/main/java/org/springframework/integration/aws/s3/InboundLocalFileOperationsImpl.java deleted file mode 100644 index 391e359..0000000 --- a/src/main/java/org/springframework/integration/aws/s3/InboundLocalFileOperationsImpl.java +++ /dev/null @@ -1,237 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.aws.s3; - -import java.io.BufferedInputStream; -import java.io.BufferedOutputStream; -import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.List; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.springframework.util.Assert; -import org.springframework.util.FileCopyUtils; -import org.springframework.util.StringUtils; - -/** - * The Implementation class for the {@link InboundLocalFileOperations} - * - * @author Amol Nayak - * - * @since 0.5 - * - */ -public class InboundLocalFileOperationsImpl implements - InboundLocalFileOperations { - - private final Log logger = LogFactory.getLog(getClass()); - - private final List handlers = new ArrayList(); - - private volatile String tempFileSuffix = ".writing"; - - private volatile boolean createDirectoriesIfRequired; - - /* (non-Javadoc) - * @see org.springframework.integration.aws.s3.InboundLocalFileOperations#addEventListener(org.springframework.integration.aws.s3.FileEventHandler) - */ - @Override - public void addEventListener(FileEventHandler handler) { - Assert.notNull(handler, "Handler instance must non null"); - handlers.add(handler); - } - - /* (non-Javadoc) - * @see org.springframework.integration.aws.s3.InboundLocalFileOperations#setEventListeners(java.util.List) - */ - @Override - public void setEventListeners(List handlers) { - Assert.notNull(handlers, "Handlers must be non null and non empty"); - Assert.notEmpty(handlers, "Handlers must be non null and non empty"); - this.handlers.clear(); - this.handlers.addAll(handlers); - } - - /* (non-Javadoc) - * @see org.springframework.integration.aws.s3.InboundLocalFileOperations#setTemporaryFileSuffix(java.lang.String) - */ - @Override - public void setTemporaryFileSuffix(String tempFileSuffix) { - if(!StringUtils.hasText(tempFileSuffix)) { - return; - } - - if(!tempFileSuffix.startsWith(".")) { - this.tempFileSuffix = "." + tempFileSuffix; - } - else { - this.tempFileSuffix = tempFileSuffix; - } - } - - /** - * Returns true if create directories if required flag is set to true - */ - public boolean isCreateDirectoriesIfRequired() { - return createDirectoriesIfRequired; - } - - /** - * Sets the flag to true if directories given are to be created if not present - * - * @param createDirectoriesIfRequired - */ - @Override - public void setCreateDirectoriesIfRequired(boolean createDirectoriesIfRequired) { - this.createDirectoriesIfRequired = createDirectoriesIfRequired; - } - - - - /* (non-Javadoc) - * @see org.springframework.integration.aws.s3.InboundLocalFileOperations#writeToFile(java.io.File, java.lang.String, java.io.InputStream) - */ - public void writeToFile(File directory, String fileName, InputStream in) - throws IOException { - Assert.notNull(directory, "Provide a non null directory"); - Assert.hasText(fileName, "Provide a non null non empty file name"); - Assert.notNull(in,"Provide a non null instance of InputStream"); - Assert.isTrue(!directory.exists() || directory.isDirectory(),"Provided directory is not a directory"); - Assert.isTrue(createDirectoriesIfRequired || directory.exists(),"Provided directories does not exist and create directory flag is false"); - - if(!directory.exists() && createDirectoriesIfRequired) { - if(!directory.mkdirs()) { - throw new IOException(String.format("Unable to create the directory '%s'", directory.getAbsolutePath())); - } - } - - if(!(in instanceof ByteArrayInputStream) - && !(in instanceof BufferedInputStream)) { - in = new BufferedInputStream(in); - } - String tempFileName = fileName + tempFileSuffix; - byte[] bytes = new byte[4096]; //4K - - String absoluteDirectoryPath = directory.getAbsolutePath(); - String filePath; - if(absoluteDirectoryPath.endsWith(File.separator)) { - filePath = absoluteDirectoryPath + tempFileName; - } - else { - filePath = absoluteDirectoryPath + File.separator + tempFileName; - } - - final File fileToWrite = new File(filePath); - if(!fileToWrite.exists()) { - fileToWrite.createNewFile(); - } - FileOutputStream fos = new FileOutputStream(fileToWrite); - BufferedOutputStream bos = new BufferedOutputStream(fos); - for(int read = 0;(read = in.read(bytes)) != -1;) { - bos.write(bytes, 0, read); - } - bos.close(); - //Now rename the file - final File dest = new File(filePath.substring(0, filePath.indexOf(tempFileSuffix))); - //ifDestination file exists, delete it - final boolean isSuccessful; - if(dest.exists()) { - boolean isDeleteSuccessful = dest.delete(); - if(isDeleteSuccessful) { - if(logger.isDebugEnabled()) { - logger.debug("Delete of file " + dest.getName() + " successful"); - } - //now rename the temp file to perm destination file - isSuccessful = renameFile(fileToWrite, dest); - } - else { - if(logger.isWarnEnabled()) { - logger.warn("Deletion of file " + dest.getName() + " not successful, falling back to overwriting the contents"); - } - FileCopyUtils.copy(fileToWrite, dest); - boolean deleteTemp = fileToWrite.delete(); - if(!deleteTemp && logger.isWarnEnabled()) { - logger.warn("Deletion of " + fileToWrite.getName() + " unsuccessful"); - } - isSuccessful = true; //as copy has occurred successfully - - } - } - else { - isSuccessful = renameFile(fileToWrite, dest); - } - //notify the listeners - if(!handlers.isEmpty()) { - FileEvent event = new FileEvent() { - - - public FileOperationType getFileOperation() { - return FileOperationType.CREATE; - } - - - public File getFile() { - if(isSuccessful) { - return dest; - } - else { - return fileToWrite; - } - } - }; - for(FileEventHandler handler:handlers) { - try { - handler.onEvent(event); - } catch (Exception e) { - if(logger.isInfoEnabled()) - logger.info("Exception occurred while notifying the handler class " - + handler.getClass().getName(), e); - } - } - } - } - - /** - * Private helper method that is used to rename the source to destination file - * - * @param fileToWrite - * @param dest - */ - private boolean renameFile(final File from, final File to) { - final boolean isSuccessful; - final boolean isRenameSuccessful = from.renameTo(to); - if(isRenameSuccessful) { - if(logger.isDebugEnabled()) { - logger.debug("Renaming of file " + from.getName() + " to " - + to.getName() + " successful"); - } - isSuccessful = isRenameSuccessful; - } - else { - if(logger.isWarnEnabled()) { - logger.warn("Renaming of file " + from.getName() + " to " - + to.getName() + " unsuccessful"); - } - isSuccessful = false; - } - return isSuccessful; - } -} diff --git a/src/main/java/org/springframework/integration/aws/s3/RegexFileNameFilter.java b/src/main/java/org/springframework/integration/aws/s3/RegexFileNameFilter.java deleted file mode 100644 index a7a4154..0000000 --- a/src/main/java/org/springframework/integration/aws/s3/RegexFileNameFilter.java +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.aws.s3; - -import org.apache.commons.io.filefilter.IOFileFilter; -import org.apache.commons.io.filefilter.RegexFileFilter; -import org.springframework.util.Assert; - -/** - * Filters out the files by matching the given File name against the given regex - * Uses Apache Commons IO {@link RegexFileFilter} internally - * - * @author Amol Nayak - * - * @since 0.5 - * - */ -public class RegexFileNameFilter extends AbstractFileNameFilter { - private final IOFileFilter filter; - - /** - * Default constructor accepting the regex - * @param regex - */ - public RegexFileNameFilter(String regex) { - Assert.hasText(regex, "Regex should be non null, non empty String"); - filter = new RegexFileFilter(regex); - } - - - /* (non-Javadoc) - * @see org.springframework.integration.aws.s3.FileNameFilter#accept(java.lang.String) - */ - - @Override - public boolean isFileNameAccepted(String fileName) { - return filter.accept(null, fileName); - } - -} diff --git a/src/main/java/org/springframework/integration/aws/s3/WildcardFileNameFilter.java b/src/main/java/org/springframework/integration/aws/s3/WildcardFileNameFilter.java deleted file mode 100644 index db0e671..0000000 --- a/src/main/java/org/springframework/integration/aws/s3/WildcardFileNameFilter.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.aws.s3; - -import org.apache.commons.io.IOCase; -import org.apache.commons.io.filefilter.IOFileFilter; -import org.apache.commons.io.filefilter.WildcardFileFilter; -import org.springframework.util.Assert; - -/** - * Performs wildcard filename filtering based on the wildcard String passed. - * Used Apache Commons IO {@link WildcardFileFilter} to perform the filtering - * - * @author Amol Nayak - * - * @since 0.5 - * - */ -public class WildcardFileNameFilter extends AbstractFileNameFilter { - - private final IOFileFilter filter; - - /** - * Default construtor accepting the wildcard string - * - * @param wildcardString - */ - public WildcardFileNameFilter(String wildcardString) { - Assert.hasText(wildcardString, "Wildcard string should be non null, non empty String"); - filter = new WildcardFileFilter(wildcardString,IOCase.INSENSITIVE); - //Our checks will be case insensitive - } - - /* (non-Javadoc) - * @see org.springframework.integration.aws.s3.FileNameFilter#accept(java.lang.String) - */ - - @Override - public boolean isFileNameAccepted(String fileName) { - return filter.accept(null, fileName); - } -} diff --git a/src/main/java/org/springframework/integration/aws/s3/config/xml/AmazonS3InboundChannelAdapterParser.java b/src/main/java/org/springframework/integration/aws/s3/config/xml/AmazonS3InboundChannelAdapterParser.java deleted file mode 100644 index 01f1129..0000000 --- a/src/main/java/org/springframework/integration/aws/s3/config/xml/AmazonS3InboundChannelAdapterParser.java +++ /dev/null @@ -1,100 +0,0 @@ -/* - * Copyright 2002-2016 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.aws.s3.config.xml; - -import org.w3c.dom.Element; - -import org.springframework.beans.BeanMetadataElement; -import org.springframework.beans.factory.BeanDefinitionStoreException; -import org.springframework.beans.factory.config.BeanDefinition; -import org.springframework.beans.factory.support.BeanDefinitionBuilder; -import org.springframework.beans.factory.xml.ParserContext; -import org.springframework.integration.aws.config.xml.AwsParserUtils; -import org.springframework.integration.aws.s3.AmazonS3InboundSynchronizationMessageSource; -import org.springframework.integration.config.xml.AbstractPollingInboundChannelAdapterParser; -import org.springframework.integration.config.xml.IntegrationNamespaceUtils; -import org.springframework.util.StringUtils; - - -/** - * The channel adapter parser for the S3 inbound parser - * - * @author Amol Nayak - * @author Li Wang - * - * @since 0.5 - * - */ -public class AmazonS3InboundChannelAdapterParser extends - AbstractPollingInboundChannelAdapterParser { - - private static final String S3_BUCKET = "bucket"; - private static final String TEMPORARY_SUFFIX = "temporary-suffix"; - private static final String S3_OPERATIONS = "s3-operations"; - private static final String AWS_ENDPOINT = "aws-endpoint"; - private static final String REMOTE_DIRECTORY = "remote-directory"; - private static final String LOCAL_DIRECTORY = "local-directory"; - private static final String LOCAL_DIRECTORY_EXPRESSION = "local-directory-expression"; - private static final String AWS_CREDENTIAL = "credentials"; - private static final String MAX_OBJECTS_PER_BATCH = "max-objects-per-batch"; - private static final String FILE_NAME_WILDCARD = "file-name-wildcard"; - private static final String FILE_NAME_REGEX = "file-name-regex"; - private static final String ACCEPT_SUB_FOLDERS = "accept-sub-folders"; - - /* (non-Javadoc) - * @see org.springframework.integration.config.xml.AbstractPollingInboundChannelAdapterParser#parseSource(org.w3c.dom.Element, org.springframework.beans.factory.xml.ParserContext) - */ - @Override - protected BeanMetadataElement parseSource(Element element, - ParserContext parserContext) { - String awsCredentials = AwsParserUtils.getAmazonWSCredentials(element, parserContext); - BeanDefinitionBuilder builder = BeanDefinitionBuilder - .genericBeanDefinition(AmazonS3InboundSynchronizationMessageSource.class); - builder.addPropertyReference(AWS_CREDENTIAL, awsCredentials); - IntegrationNamespaceUtils.setReferenceIfAttributeDefined(builder, element, S3_OPERATIONS); - IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element, S3_BUCKET); - IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element, TEMPORARY_SUFFIX); - IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element, REMOTE_DIRECTORY); - IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element, AWS_ENDPOINT); - String directory = element.getAttribute(LOCAL_DIRECTORY); - String directoryExpression = element.getAttribute(LOCAL_DIRECTORY_EXPRESSION); - boolean hasDirectory = StringUtils.hasText(directory); - boolean hasDirectoryExpression = StringUtils.hasText(directoryExpression); - if(!hasDirectory && !hasDirectoryExpression) { - String message = - String.format("One of attributes '%s' and '%s' is required", LOCAL_DIRECTORY, LOCAL_DIRECTORY_EXPRESSION); - throw new BeanDefinitionStoreException(message); - } - if(hasDirectory && hasDirectoryExpression) { - String message = - String.format("Attributes '%s' and '%s' are mutually exclusive to each other", LOCAL_DIRECTORY, LOCAL_DIRECTORY_EXPRESSION); - throw new BeanDefinitionStoreException(message); - } - else { - BeanDefinition expressionDef = - IntegrationNamespaceUtils.createExpressionDefinitionFromValueOrExpression("local-directory", - "local-directory-expression", parserContext, element, true); - builder.addPropertyValue("directory", expressionDef); - } - IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element, MAX_OBJECTS_PER_BATCH); - IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element, ACCEPT_SUB_FOLDERS); - IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element, FILE_NAME_WILDCARD); - IntegrationNamespaceUtils.setValueIfAttributeDefined(builder, element, FILE_NAME_REGEX); - return builder.getBeanDefinition(); - } - -} diff --git a/src/main/java/org/springframework/integration/aws/s3/core/AbstractAmazonS3Operations.java b/src/main/java/org/springframework/integration/aws/s3/core/AbstractAmazonS3Operations.java deleted file mode 100644 index 76404a3..0000000 --- a/src/main/java/org/springframework/integration/aws/s3/core/AbstractAmazonS3Operations.java +++ /dev/null @@ -1,503 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.aws.s3.core; - -import java.io.BufferedInputStream; -import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.ArrayList; -import java.util.List; -import java.util.Map; - -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; - -import org.springframework.beans.factory.InitializingBean; -import org.springframework.integration.aws.core.AWSCommonUtils; -import org.springframework.integration.aws.core.AWSCredentials; -import org.springframework.util.Assert; - -import com.amazonaws.util.BinaryUtils; - -/** - * The common super class for any implementation of {@link AmazonS3Operations}. The sub class - * has to implement all the functionality for performing the actual work to add, remove, update - * delete, list the objects in an S3 bucket - * - * @author Amol Nayak - * - * @since 0.5 - * - */ -public abstract class AbstractAmazonS3Operations implements AmazonS3Operations, InitializingBean { - - protected final Log logger = LogFactory.getLog(getClass()); - - private volatile long multipartUploadThreshold; - - private volatile File temporaryDirectory = new File(System.getProperty("java.io.tmpdir")); - - private volatile String temporaryFileSuffix = ".writing"; - - public final String PATH_SEPARATOR = "/"; - - private final AWSCredentials credentials; - - private String awsEndpoint; - - - /** - * The constructor that accepts the {@link AWSCredentials} - * @param credentials - */ - protected AbstractAmazonS3Operations(AWSCredentials credentials) { - Assert.notNull(credentials, "null 'credentials' provided"); - this.credentials = credentials; - } - - - /** - * Get the threshold value in bytes above which multi part upload will be used - */ - public long getMultipartUploadThreshold() { - return multipartUploadThreshold; - } - - - /** - * The threshold value in bytes above which the service will use multi part upload. - * All the uploads below this value will be uploaded in a single thread - * Minimum value for the threshold is 5120 Bytes (5 KB). - * It is recommended by Amazon to use Multi part uploads for all the uploads - * above 100 MB - * If the value is set to a number above Integer.MAX_VALUE, the value will be - * set to Integer.MAX_VALUE. - * - * @param multipartUploadThreshold - */ - public void setMultipartUploadThreshold(long multipartUploadThreshold) { - Assert.isTrue(multipartUploadThreshold >= 5120, - "Minimum threshold for multipart upload is 5120 bytes"); - this.multipartUploadThreshold = multipartUploadThreshold; - } - - - /** - * Gets the temporary directory - */ - public File getTemporaryDirectory() { - return temporaryDirectory; - } - - /** - * The temporary directory that will be used to write the files received over stream - * @param temporaryDirectory - */ - public void setTemporaryDirectory(File temporaryDirectory) { - Assert.notNull(temporaryDirectory, "Provided temporaryDirectory is null"); - Assert.isTrue(temporaryDirectory.exists(), "The given temporary directory does not exist"); - Assert.isTrue(temporaryDirectory.isDirectory(), "The given temporary directory path has to be a directory"); - this.temporaryDirectory = temporaryDirectory; - } - - - /** - * The temporary directory that will be used to write the files received over stream - * @param temporaryDirectory - */ - public void setTemporaryDirectory(String temporaryDirectory) { - Assert.hasText(temporaryDirectory, "Provided temporary directory string is null or empty string"); - setTemporaryDirectory(new File(temporaryDirectory)); - } - - - /** - * Gets the temporary file suffix that is appended to the file while writing to - * the temporary directory - */ - public String getTemporaryFileSuffix() { - return temporaryFileSuffix; - } - - - /** - * Gets the temporary file suffix - * @param temporaryFileSuffix - */ - public void setTemporaryFileSuffix(String temporaryFileSuffix) { - Assert.hasText(temporaryFileSuffix, "The temporary file suffix must not be null or empty"); - if (!temporaryFileSuffix.startsWith(".")) { - temporaryFileSuffix = "." + temporaryFileSuffix; - } - - this.temporaryFileSuffix = temporaryFileSuffix; - } - - /** - * Gets the AWS endpoint to use for all the operations, by default if none is set then us-east-1 is - * assumed. - * - */ - public String getAwsEndpoint() { - return awsEndpoint; - } - - - /** - * Sets the valid AWS endpoint to be used by the client to connect to appropriate - * region to perform the operations - * - * @param awsEndpoint - */ - public void setAwsEndpoint(String awsEndpoint) { - Assert.hasText(awsEndpoint, "Given AWS Endpoint has to be non null and non empty string"); - this.awsEndpoint = awsEndpoint; - } - - - /** - * The implemented afterPropertiesSet method - */ - public final void afterPropertiesSet() throws Exception { - //TODO: protect by lock? - init(); - } - - /** - * The subclass needs to override this method if it desires to perform any initializing - * of the class - */ - protected void init() { - - } - - /** - * Reads the stream provided and writes the file to the temp location - * @param in the Stream from which the data of the Object is to be read - * @param objectName the name of the object that would be used to upload the file - */ - private File getTempFile(InputStream in, String bucketName, String objectName) { - InputStream inStream; - if (!(in instanceof BufferedInputStream) && !(in instanceof ByteArrayInputStream)) { - inStream = new BufferedInputStream(in); - } - else { - inStream = in; - } - String fileName; - if (objectName.contains(PATH_SEPARATOR)) { - String[] splits = objectName.split(PATH_SEPARATOR); - fileName = splits[splits.length - 1]; - } - else { - fileName = objectName; - } - File temporaryDirectory = getTemporaryDirectory(); - String temporaryFileSuffix = getTemporaryFileSuffix(); - - String filePath = temporaryDirectory.getAbsoluteFile() + File.separator + fileName + temporaryFileSuffix; - - if (logger.isDebugEnabled()) { - logger.debug("Temporary file path is " + filePath); - } - - //Write data to temporary file - File tempFile = new File(filePath); - FileOutputStream fos = null; - try { - fos = new FileOutputStream(tempFile); - byte[] bytes = new byte[1024]; - int read = 0; - while (true) { - read = inStream.read(bytes); - if (read == -1) { - break; - } - fos.write(bytes, 0, read); - } - } - catch (FileNotFoundException e) { - throw new AmazonS3OperationException(credentials.getAccessKey(), - bucketName, - objectName, - "Exception caught while writing the temporary file from input stream", e); - } - catch (IOException ioe) { - throw new AmazonS3OperationException(credentials.getAccessKey(), - bucketName, - objectName, - "Exception caught while reading from the provided input stream", ioe); - } - finally { - if (fos != null) { - try { - fos.flush(); - fos.close(); - } - catch (IOException e) { - //just log - logger.error("Unable to close the stream to the temp file being created", e); - } - try { - in.close(); - } - catch (IOException e) { - //just log - logger.error("Unable to close the input stream source", e); - } - } - } - return tempFile; - } - - - /** - * The implementation of the core common operations that would be performed - * before and after the subclass does the actual work of putting the object to - * the given bucket - * - * @param bucketName The bucket to which the object is to be put - * @param folder The folder to which the object is to be uploaded - * @param objectName The name of the object in the given bucket - * @param s3Object The {@link AmazonS3Object} instance that represents the object - * to be uploaded. - */ - @Override - public final void putObject(String bucketName, String folder, String objectName, - AmazonS3Object s3Object) { - Assert.hasText(bucketName, "null or empty bucketName provided"); - Assert.hasText(objectName, "null or empty object name provided"); - Assert.notNull(s3Object, "null s3 object provided for upload"); - - - File file = s3Object.getFileSource(); - InputStream in = s3Object.getInputStream(); - Assert.isTrue(file != null ^ in != null, - "Exactly one of file or inpuut stream needed in the provided s3 object"); - - boolean isTempFile = false; - if (in != null) { - //We don't know the source of the stream and hence we read the content - //and write to the temporary file. - file = getTempFile(in, bucketName, objectName); - isTempFile = true; - } - - String key = getKeyFromFolder(folder, objectName); - - //if the size of the file is greater than the threshold for multipart upload, - //set the Content-MD5 header for this upload. This header will also come handy - //later in inbound-channel-adapter where we cant find the MD5 sum of the - //multipart upload file from its ETag - - String stringContentMD5 = BinaryUtils.toBase64(AWSCommonUtils.getContentsMD5AsBytes(file)); - - try { - doPut(bucketName, key, file, s3Object.getObjectACL(), - s3Object.getUserMetaData(), stringContentMD5); - } - catch (Exception e) { - throw new AmazonS3OperationException( - credentials.getAccessKey(), bucketName, - key, - "Encountered exception while putting an object, see root cause for more details", - e); - - } - - if (isTempFile) { - //Delete the temp file - if (logger.isDebugEnabled()) { - logger.debug("Deleting temp file: " + file.getName()); - } - boolean deleteSuccessful = file.delete(); - if (!deleteSuccessful) { - logger.warn("Unable to delete file '" + file.getName() + "'"); - } - } - } - - - /** - * The private method that takes the folder and the object name as the parameters and - * constructs the key that would be for the item to be retrieved or put in the bucket - * - * @param folder - * @param objectName - */ - private String getKeyFromFolder(String folder, String objectName) { - if (objectName.startsWith(PATH_SEPARATOR)) { - //remove the leading / of the object name - objectName = objectName.substring(1); - } - String key; - if (folder != null) { - key = folder.endsWith(PATH_SEPARATOR) ? - folder + objectName : folder + PATH_SEPARATOR + objectName; - } - else { - key = objectName; - } - - //check if the foldername begins with a /, if yes, remove it as well as it created - //one directory with blank name - - if (key.startsWith(PATH_SEPARATOR)) { - key = key.substring(1); - } - return key; - } - - /** - * Deletes the object with the given name from the provided bucket and folder. - * - */ - public boolean removeObject(String bucketName, String folder, - String objectName) { - throw new UnsupportedOperationException("Operation not et supported"); - } - - /** - * List the objects in a given bucket in the given folder. - * - * @param bucketName The bucket in which we want to list the objects in - * @param nextMarker The number of objects can be very large and this serves as the marker - * for remembering the last record fetch in the last retrieve operation. - * @param pageSize The max number of records to be retrieved in one list object operation. - * - */ - public final PaginatedObjectsView listObjects(String bucketName, String folder, String nextMarker, int pageSize) { - Assert.hasText(bucketName, "Bucket name should be non null and non empty"); - Assert.isTrue(pageSize >= 0, "Page size should be a non negative number"); - - if (logger.isDebugEnabled()) { - logger.debug("Listing objects from bucket " + bucketName + " and folder " + folder); - logger.debug("Next marker is " + nextMarker + " and pageSize is " + pageSize); - } - - - String prefix = null; - if (folder != null && !PATH_SEPARATOR.equals(folder)) { - prefix = folder; - } - //check if the prefix begins with / - if (prefix != null && prefix.startsWith(PATH_SEPARATOR)) { - prefix = prefix.substring(1); - } - - return doListObjects(bucketName, nextMarker, pageSize, prefix); - } - - /** - * Gets the object from the given bucket, folder and the name. - * - * @param bucketName The bucket from which to retrieve the object - * @param folder The folder name - * @param objectName The name of the object to retrieve - */ - public final AmazonS3Object getObject(String bucketName, String folder, - String objectName) { - Assert.hasText(bucketName, "Bucket name should be non null and non empty"); - if (logger.isDebugEnabled()) { - logger.debug("Getting from bucket " + bucketName + - ", from folder " + folder + " the object name " + objectName); - } - String key = getKeyFromFolder(folder, objectName); - try { - return doGetObject(bucketName, key); - } - catch (Exception e) { - throw new AmazonS3OperationException( - credentials.getAccessKey(), bucketName, - key, - "Encountered exception while putting an object, see root cause for more details", - e); - } - } - - /** - * Gets the object from the given bucket with the given key using the AWS SDK implementation - * - * @param bucketName - * @param key - */ - protected abstract AmazonS3Object doGetObject(String bucketName, String key); - - - /** - * The implementation should use the appropriate API to list objects from the given bucket - * - * @param bucketName The bucket in which we want to list the objects in - * @param nextMarker The number of objects can be very large and this serves as the marker - * for remembering the last record fetch in the last retrieve operation. - * @param pageSize The max number of records to be retrieved in one list object operation. - * @param prefix The prefix for the list operation, this can serve as the folder whose contents - * are to be listed. - */ - protected abstract PaginatedObjectsView doListObjects(String bucketName, - String nextMarker, int pageSize, String prefix); - - - /** - * The abstract method to be implemented by the subclass that would be doing the job - * of uploading the given file against the given key in the given bucket - * - * @param bucket The bucket on S3 where this object is to be put - * @param key The key against which this Object is to be stored in S3 - * @param file resource to be uploaded to S3 - * @param objectACL the Object's Access controls for the object to be uploaded - * @param userMetadata The user's metadata to be associated with the object uploaded - * @param stringContentMD5 The MD5 sum of the contents of the file to be uploaded - * - */ - protected abstract void doPut(String bucket, String key, File file, - AmazonS3ObjectACL objectACL, Map userMetadata, String stringContentMD5); - - - static class PagninatedObjectsViewImpl implements PaginatedObjectsView { - - private final List objectSummary; - - private final String nextMarker; - - public PagninatedObjectsViewImpl(List objectSummary, - String nextMarker) { - this.objectSummary = objectSummary; - this.nextMarker = nextMarker; - } - - - public List getObjectSummary() { - return objectSummary != null ? objectSummary : new ArrayList(); - } - - - public boolean hasMoreResults() { - return nextMarker != null; - } - - - public String getNextMarker() { - return nextMarker; - } - - } - -} diff --git a/src/main/java/org/springframework/integration/aws/s3/core/AmazonS3Object.java b/src/main/java/org/springframework/integration/aws/s3/core/AmazonS3Object.java deleted file mode 100644 index 4cdf557..0000000 --- a/src/main/java/org/springframework/integration/aws/s3/core/AmazonS3Object.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.aws.s3.core; - -import java.io.File; -import java.io.InputStream; -import java.io.Serializable; -import java.util.Collections; -import java.util.Map; - -import org.springframework.util.Assert; - - - -/** - * The Amazon S3 Object representing the Object in S3 - * - * @author Amol Nayak - * - * @since 0.5 - * - */ -public class AmazonS3Object implements Serializable { - - /** - * - */ - private static final long serialVersionUID = -832622119907619624L; - - private Map userMetaData; - - private Map metaData; - - private final InputStream inputStream; - - private final File fileSource; - - private final AmazonS3ObjectACL objectACL; - - /** - * The default constructor - * - * @param userMetaData - * @param metaData - * @param inputStream - */ - public AmazonS3Object(Map userMetaData, - Map metaData, InputStream inputStream,File fileSource,AmazonS3ObjectACL objectACL) { - if(userMetaData != null) - this.userMetaData = Collections.unmodifiableMap(userMetaData); - - if(metaData != null) - this.metaData = Collections.unmodifiableMap(metaData); - - Assert.isTrue((inputStream == null) ^ (fileSource == null), - "Exactly one of 'inputStream' or 'fileSource' must be provided"); - - this.inputStream = inputStream; - this.fileSource = fileSource; - this.objectACL = objectACL; - } - - /** - * The constructor that delegates to {@link #AmazonS3Object(Map, Map, InputStream, File, AmazonS3ObjectACL)} - * with null {@link AmazonS3ObjectACL} - * - * @param userMetaData - * @param metaData - * @param inputStream - * @param fileSource - */ - public AmazonS3Object(Map userMetaData, - Map metaData, InputStream inputStream,File fileSource) { - this(userMetaData,metaData,inputStream,fileSource,null); - } - - - /** - * Gets the User Metadata associated with given Amazon S3 object - */ - public Map getUserMetaData() { - return userMetaData; - } - - /** - * Gets the Metadata associated with the given Amazon S3 object - */ - public Map getMetaData() { - return metaData; - } - - /** - * Gets the {@link InputStream} to the resource - */ - public InputStream getInputStream() { - return inputStream; - } - - /** - * Gets the file source - */ - public File getFileSource() { - return fileSource; - } - - /** - * Gets the Access controls associated with the S3 Object - */ - public AmazonS3ObjectACL getObjectACL() { - return objectACL; - } -} diff --git a/src/main/java/org/springframework/integration/aws/s3/core/AmazonS3ObjectACL.java b/src/main/java/org/springframework/integration/aws/s3/core/AmazonS3ObjectACL.java deleted file mode 100644 index 6b3aabb..0000000 --- a/src/main/java/org/springframework/integration/aws/s3/core/AmazonS3ObjectACL.java +++ /dev/null @@ -1,62 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.aws.s3.core; - -import java.util.HashSet; -import java.util.Set; - -import org.springframework.util.Assert; - -/** - * The object containing the Amazon S3 Object's ACL. Access is used to control the access to - * the resource in S3 bucket - * - * @author Amol Nayak - * - * @since 0.5 - * - */ -public class AmazonS3ObjectACL { - - private Set grants = new HashSet(); - - /** - * Gets all the grants on the object in the bucket - */ - public Set getGrants() { - return grants; - } - - /** - * Sets the provided grants on the S3 object - * - * @param grants - */ - public void setGrants(Set grants) { - Assert.notNull(grants, "Provide non null 'grants'"); - this.grants = grants; - } - - /** - * A convenience method that will be used to add grants to the objects. - * - * @param grant - */ - public void addGrant(ObjectGrant grant) { - Assert.notNull(grant,"Provide non null object grant"); - grants.add(grant); - } -} diff --git a/src/main/java/org/springframework/integration/aws/s3/core/AmazonS3OperationException.java b/src/main/java/org/springframework/integration/aws/s3/core/AmazonS3OperationException.java deleted file mode 100644 index d1867e2..0000000 --- a/src/main/java/org/springframework/integration/aws/s3/core/AmazonS3OperationException.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.aws.s3.core; - -import org.springframework.integration.aws.core.AWSOperationException; - -/** - * A subclass of {@link AWSOperationException} which indicates a failure in performing - * an operation on the object in S3. - * - * @author Amol Nayak - * - * @since 0.5 - * - */ -public class AmazonS3OperationException extends AWSOperationException { - - /** - * - */ - private static final long serialVersionUID = 9518185510906801L; - private final String bucket; - private final String objectName; - - /** - *The constructor that instantiates with the Account's access key the bucket name - *and the object name - * - * @param accessKey - * @param bucket - * @param objectName - */ - public AmazonS3OperationException(String accessKey, String bucket, - String objectName) { - super(accessKey); - this.bucket = bucket; - this.objectName = objectName; - } - - - - /** - * The constructor that instantiates with the Account's access key the bucket name - * the object name, the exception message and the actual exception - * - * @param accessKey - * @param bucket - * @param objectName - * @param message - * @param cause - */ - public AmazonS3OperationException(String accessKey, String bucket, - String objectName,String message, - Throwable cause) { - super(accessKey, message, cause); - this.bucket = bucket; - this.objectName = objectName; - } - - /** - * The constructor that instantiates with the Account's access key the bucket name - * the object name, the exception message - * - * @param accessKey - * @param bucket - * @param objectName - * @param message - */ - public AmazonS3OperationException(String accessKey, String bucket, - String objectName,String message) { - super(accessKey, message); - this.bucket = bucket; - this.objectName = objectName; - } - - /** - * The constructor that instantiates with the Account's access key the bucket name - * the object name, the root cause - * - * @param accessKey - * @param bucket - * @param objectName - * @param cause - */ - public AmazonS3OperationException(String accessKey, String bucket, - String objectName,Throwable cause) { - super(accessKey, cause); - this.bucket = bucket; - this.objectName = objectName; - } - - - /** - * Gets the bucket name for which an S3 operation failed - */ - public String getBucket() { - return bucket; - } - - /** - * Gets the object name where an S3 operation failed - */ - public String getObjectName() { - return objectName; - } -} diff --git a/src/main/java/org/springframework/integration/aws/s3/core/AmazonS3Operations.java b/src/main/java/org/springframework/integration/aws/s3/core/AmazonS3Operations.java deleted file mode 100644 index 13391d7..0000000 --- a/src/main/java/org/springframework/integration/aws/s3/core/AmazonS3Operations.java +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.aws.s3.core; - -/** - * The Core interface for performing various operations on Amazon S3 like listing objects - * in the bucket, get an object, put an object and remove an object - * - * @author Amol Nayak - * - * @since 0.5 - * - */ -public interface AmazonS3Operations { - - public static final String CONTENT_MD5_HEADER = "Content-MD5"; - - /** - * Lists Objects in the given bucket and given folder. Provide / if you - * wish to list objects at the root of the bucket - * - * @param bucketName - * @param folder - * @param nextMarker - * @param pageSize - * @return the {@link PaginatedObjectsView} of the matching result - */ - PaginatedObjectsView listObjects(String bucketName,String folder,String nextMarker,int pageSize); - - /** - * Put the given {@link AmazonS3Object} in the provided bucket in the folder specified with the name given - * The object if exists, will be overwritten and the folder path hierarchy - * if absent will be created - * - * @param bucketName - * @param folder - * @param objectName - * @param s3Object - */ - void putObject(String bucketName,String folder,String objectName,AmazonS3Object s3Object); - - /** - * Gets the Object from Amazon S3 from the specified bucket,folder and with - * the given objectName - * - * @param bucketName - * @param folder - * @param objectName - * @return The S3 object corresponding to the given details. Null if no object found - */ - AmazonS3Object getObject(String bucketName,String folder,String objectName); - - /** - * Removes the specified object from the bucket given, folder specified - * and the given object name from S3 - * @param bucketName - * @param folder - * @param objectName - * @return true if the object was successfully removed else false - */ - boolean removeObject(String bucketName,String folder,String objectName); -} diff --git a/src/main/java/org/springframework/integration/aws/s3/core/DefaultAmazonS3Operations.java b/src/main/java/org/springframework/integration/aws/s3/core/DefaultAmazonS3Operations.java deleted file mode 100644 index fa366c0..0000000 --- a/src/main/java/org/springframework/integration/aws/s3/core/DefaultAmazonS3Operations.java +++ /dev/null @@ -1,351 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.aws.s3.core; - -import java.io.File; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; -import java.util.Map; -import java.util.concurrent.ThreadPoolExecutor; - -import org.springframework.integration.aws.core.AWSCredentials; -import org.springframework.integration.aws.core.AbstractAWSClientFactory; -import org.springframework.util.Assert; - -import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.services.s3.AmazonS3Client; -import com.amazonaws.services.s3.model.AccessControlList; -import com.amazonaws.services.s3.model.AmazonS3Exception; -import com.amazonaws.services.s3.model.CanonicalGrantee; -import com.amazonaws.services.s3.model.EmailAddressGrantee; -import com.amazonaws.services.s3.model.GetObjectRequest; -import com.amazonaws.services.s3.model.GroupGrantee; -import com.amazonaws.services.s3.model.ListObjectsRequest; -import com.amazonaws.services.s3.model.ObjectListing; -import com.amazonaws.services.s3.model.ObjectMetadata; -import com.amazonaws.services.s3.model.Permission; -import com.amazonaws.services.s3.model.PutObjectRequest; -import com.amazonaws.services.s3.model.S3Object; -import com.amazonaws.services.s3.transfer.TransferManager; -import com.amazonaws.services.s3.transfer.TransferManagerConfiguration; -import com.amazonaws.services.s3.transfer.Upload; - - -/** - * The default, out of the box implementation of the {@link AmazonS3Operations} that is implemented - * using AWS SDK. - * - * @author Amol Nayak - * - * @since 0.5 - * - */ -public class DefaultAmazonS3Operations extends AbstractAmazonS3Operations { - - private final AWSCredentials credentials; - - private AmazonS3Client client; - - private volatile TransferManager transferManager; //Used to upload to S3 - - private volatile ThreadPoolExecutor threadPoolExecutor; - - private volatile AbstractAWSClientFactory s3Factory; - - /** - * Constructor - * @param credentials - */ - public DefaultAmazonS3Operations(final AWSCredentials credentials) { - super(credentials); - this.credentials = credentials; - s3Factory = new AbstractAWSClientFactory() { - @Override - protected AmazonS3Client getClientImplementation() { - String accessKey = credentials.getAccessKey(); - String secretKey = credentials.getSecretKey(); - BasicAWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey); - return new AmazonS3Client(credentials); - } - - }; - } - - @Override - protected void init() { - - client = s3Factory.getClient(getAwsEndpoint()); - - if(threadPoolExecutor == null) { - //Will use the Default Executor, - //See com.amazonaws.services.s3.transfer.internal.TransferManagerUtils for more details - transferManager = new TransferManager(client); - } - else { - transferManager = new TransferManager(client, threadPoolExecutor); - } - - //As per amazon it is recommended to use Multi part upload above 100 MB - long multipartUploadThreshold = getMultipartUploadThreshold(); - if(multipartUploadThreshold > 0) { - TransferManagerConfiguration config = new TransferManagerConfiguration(); - if(multipartUploadThreshold > Integer.MAX_VALUE) { - config.setMultipartUploadThreshold(Integer.MAX_VALUE); //2GB - } - else { - config.setMultipartUploadThreshold((int)multipartUploadThreshold); - } - transferManager.setConfiguration(config); - } - //If none is set, we use the default - } - - /** - * The implementation that uses the AWS SDK to list objects from the given bucket - * - * @param bucketName The bucket in which we want to list the objects in - * @param nextMarker The number of objects can be very large and this serves as the marker - * for remembering the last record fetch in the last retrieve operation. - * @param pageSize The max number of records to be retrieved in one list object operation. - * @param prefix The prefix for the list operation, this can serve as the folder whose contents - * are to be listed. - */ - @Override - protected PaginatedObjectsView doListObjects(String bucketName, - String nextMarker, int pageSize, String prefix) { - - ListObjectsRequest listObjectsRequest = - new ListObjectsRequest() - .withBucketName(bucketName) - .withPrefix(prefix) - .withMarker(nextMarker); - - if(pageSize > 0) { - listObjectsRequest.withMaxKeys(pageSize); - } - - ObjectListing listing = client.listObjects(listObjectsRequest); - PaginatedObjectsView view = null; - List summaries = listing.getObjectSummaries(); - if(summaries != null && !summaries.isEmpty()) { - List objectSummaries = new ArrayList(); - for(final com.amazonaws.services.s3.model.S3ObjectSummary summary:summaries) { - S3ObjectSummary summ = new S3ObjectSummary() { - - public long getSize() { - return summary.getSize(); - } - - public Date getLastModified() { - return summary.getLastModified(); - } - - public String getKey() { - return summary.getKey(); - } - - public String getETag() { - return summary.getETag(); - } - - public String getBucketName() { - return summary.getBucketName(); - } - }; - objectSummaries.add(summ); - } - view = new PagninatedObjectsViewImpl(objectSummaries,listing.getNextMarker()); - } - return view; - } - - /** - * Gets the object from the given bucket with the given key using the AWS SDK implementation - * - * @param bucketName - * @param key - * @return The Amazon S3 Object representing the Object in S3, may be null. - */ - @Override - protected AmazonS3Object doGetObject(String bucketName, String key) { - GetObjectRequest request = new GetObjectRequest(bucketName, key); - S3Object s3Object; - try { - s3Object = client.getObject(request); - } catch (AmazonS3Exception e) { - if("NoSuchKey".equals(e.getErrorCode())) { - //If the key is not found, return null rather than throwing the exception - return null; - } - else { - //throw the exception to caller in all other cases - throw e; - } - } - return new AmazonS3Object(s3Object.getObjectMetadata().getUserMetadata(), - s3Object.getObjectMetadata().getRawMetadata(), - s3Object.getObjectContent(), - null); - } - - /** - * The implementation puts the given {@link File} instance to the provided bucket against - * the given key. - * - * @param bucketName The bucket on S3 where this object is to be put - * @param key The key against which this Object is to be stored in S3 - * @param file resource to be uploaded to S3 - * @param objectACL the Object's Access controls for the object to be uploaded - * @param userMetadata The user's metadata to be associated with the object uploaded - * @param stringContentMD5 The MD5 sum of the contents of the file to be uploaded - */ - @Override - public void doPut(String bucketName, String key, File file, AmazonS3ObjectACL objectACL, - Map userMetadata,String stringContentMD5) { - - ObjectMetadata metadata = new ObjectMetadata(); - PutObjectRequest request = new PutObjectRequest(bucketName, key, file); - - request.withMetadata(metadata); - - if(stringContentMD5 != null) { - metadata.setContentMD5(stringContentMD5); - } - - if(userMetadata != null) { - metadata.setUserMetadata(userMetadata); - } - - Upload upload; - try { - upload = transferManager.upload(request); - } catch (Exception e) { - throw new AmazonS3OperationException( - credentials.getAccessKey(), bucketName, - key, - "Encountered Exception while invoking upload on multipart/single thread file, " + - "see nested exceptions for more details", - e); - } - //Wait till the upload completes, the call to putObject is synchronous - try { - if(logger.isInfoEnabled()) { - logger.info("Waiting for Upload to complete"); - } - upload.waitForCompletion(); - if(logger.isInfoEnabled()) { - logger.info("Upload completed"); - } - } catch (Exception e) { - throw new AmazonS3OperationException( - credentials.getAccessKey(), bucketName, - key, - "Encountered Exception while uploading the multipart/single thread file, " + - "see nested exceptions for more details", - e); - } - //Now since the object is present on S3, set the AccessControl list on it - //Please note that it is not possible to set the object ACL with the - //put object request, and hence both these operations cannot be atomic - //it is possible the objects is uploaded and the ACl not set due to some - //failure - - if(objectACL != null) { - if(logger.isInfoEnabled()) { - logger.info("Setting Access control list for key " + key); - } - try { - client.setObjectAcl(bucketName, key, - getAccessControlList(bucketName, key, objectACL)); - } catch (Exception e) { - throw new AmazonS3OperationException( - credentials.getAccessKey(), bucketName, - key, - "Encountered Exception while setting the Object ACL for key , " + key + - "see nested exceptions for more details", - e); - } - if(logger.isDebugEnabled()) { - logger.debug("Successfully set the object ACL"); - } - } - } - - /** - * Gets the {@link AccessControlList} from the given {@link AmazonS3ObjectACL} - */ - private AccessControlList getAccessControlList(String bucketName,String key,AmazonS3ObjectACL acl) { - AccessControlList accessControlList = null; - if(acl != null) { - if(!acl.getGrants().isEmpty()) { - accessControlList = client.getObjectAcl(bucketName, key); - for(ObjectGrant objGrant:acl.getGrants()) { - Grantee grantee = objGrant.getGrantee(); - com.amazonaws.services.s3.model.Grantee awsGrantee; - if(grantee.getGranteeType() == GranteeType.CANONICAL_GRANTEE_TYPE) { - awsGrantee = new CanonicalGrantee(grantee.getIdentifier()); - } - else if(grantee.getGranteeType() == GranteeType.EMAIL_GRANTEE_TYPE) { - awsGrantee = new EmailAddressGrantee(grantee.getIdentifier()); - } - else { - awsGrantee = GroupGrantee.parseGroupGrantee(grantee.getIdentifier()); - if(awsGrantee == null) { - logger.warn("Group grantee with identifier: \"" + grantee.getIdentifier() + "\" not found. skipping this grant"); - continue; - } - } - ObjectPermissions perm = objGrant.getPermission(); - Permission permission; - if(perm == ObjectPermissions.READ) { - permission = Permission.Read; - } - else if(perm == ObjectPermissions.READ_ACP) { - permission = Permission.ReadAcp; - } - else - permission = Permission.WriteAcp; - - accessControlList.grantPermission(awsGrantee, permission); - } - } - } - return accessControlList; - } - - - /** - * Gets the thread pool executor that will be used to upload the object in multiparts - * concurrently - */ - public ThreadPoolExecutor getThreadPoolExecutor() { - return threadPoolExecutor; - } - - - /** - * Used only when we upload the data using multi part upload. The thread pool will be used - * to upload the data concurrently - * - * @param threadPoolExecutor May not be null - */ - public void setThreadPoolExecutor(ThreadPoolExecutor threadPoolExecutor) { - Assert.notNull(threadPoolExecutor, "'threadPoolExecutor' is null"); - this.threadPoolExecutor = threadPoolExecutor; - } - -} diff --git a/src/main/java/org/springframework/integration/aws/s3/core/Grantee.java b/src/main/java/org/springframework/integration/aws/s3/core/Grantee.java deleted file mode 100644 index 947e509..0000000 --- a/src/main/java/org/springframework/integration/aws/s3/core/Grantee.java +++ /dev/null @@ -1,103 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.aws.s3.core; - -/** - * Indicates the Grantee who is being given the Access to a particular resource on S3 - * - * @author Amol Nayak - * - * @since 0.5 - * - */ -public class Grantee { - - private String identifier; - private GranteeType granteeType; - - public Grantee() { - - } - - public Grantee(String identifier, GranteeType granteeType) { - this.identifier = identifier; - this.granteeType = granteeType; - } - - /** - * The identifier of a particular type identifying the grantee - * @param id - */ - void setIdentifier(String identifier) { - this.identifier = identifier; - } - - /** - * Gets the particular identifier representing the grantee - */ - public String getIdentifier() { - return identifier; - } - - /** - * Gets the Particular grantee Type - */ - public GranteeType getGranteeType() { - return granteeType; - } - - /** - * Sets the Type of the grantee see {@link GranteeType} for more information - * @param granteeType - */ - public void setGranteeType(GranteeType granteeType) { - this.granteeType = granteeType; - } - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result - + ((granteeType == null) ? 0 : granteeType.hashCode()); - result = prime * result - + ((identifier == null) ? 0 : identifier.hashCode()); - return result; - } - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - Grantee other = (Grantee) obj; - if (granteeType != other.granteeType) - return false; - if (identifier == null) { - if (other.identifier != null) - return false; - } else if (!identifier.equals(other.identifier)) - return false; - return true; - } - - - - -} diff --git a/src/main/java/org/springframework/integration/aws/s3/core/GranteeType.java b/src/main/java/org/springframework/integration/aws/s3/core/GranteeType.java deleted file mode 100644 index 21427b0..0000000 --- a/src/main/java/org/springframework/integration/aws/s3/core/GranteeType.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.aws.s3.core; - -/** - * Identifies the type of the grantee. E.g. A grantee can be identified using the canonical - * identifier or the email id. - * - * @author Amol Nayak - * - * @since 0.5 - * - */ -public enum GranteeType { - - /** - * This represents the Canonical id of the users AWS account - */ - CANONICAL_GRANTEE_TYPE, - - /** - * This email is usually resolved to the canonical id of the user. - * This would fail and an error would be thrown if more than 2 accounts are - * related to the user's email account. - */ - EMAIL_GRANTEE_TYPE, - - /** - * These represent come constants representing some predefined groups by Amazon S3 - */ - GROUP_GRANTEE_TYPE; -} diff --git a/src/main/java/org/springframework/integration/aws/s3/core/GroupGranteeType.java b/src/main/java/org/springframework/integration/aws/s3/core/GroupGranteeType.java deleted file mode 100644 index 11b8420..0000000 --- a/src/main/java/org/springframework/integration/aws/s3/core/GroupGranteeType.java +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.aws.s3.core; - -/** - * Various types of Groups who can be granted permissions on amazon S3 objects - * - * @author Amol Nayak - * - * @since 0.5 - * - */ -public enum GroupGranteeType { - - /** - * To grants anonymous access to all objects in the bucket - */ - AllUsers("http://acs.amazonaws.com/groups/global/AllUsers"), - /** - * To grant access to all authenticated users of AWS who is logged in using - * their AWS credentials - */ - AuthenticatedUsers("http://acs.amazonaws.com/groups/global/AuthenticatedUsers"); - - private String identifier; - - private GroupGranteeType(String identifier) { - this.identifier = identifier; - } - - public String getIdentifier() { - return identifier; - } -} diff --git a/src/main/java/org/springframework/integration/aws/s3/core/ObjectGrant.java b/src/main/java/org/springframework/integration/aws/s3/core/ObjectGrant.java deleted file mode 100644 index 7f4d8f2..0000000 --- a/src/main/java/org/springframework/integration/aws/s3/core/ObjectGrant.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.aws.s3.core; - -/** - * Represent one Grant for a Grantee and the associated permissions - * - * @author Amol Nayak - * - * @since 0.5 - * - */ -public class ObjectGrant { - - private final Grantee grantee; - private final ObjectPermissions permission; - - - /** - * Instantiate an Object grant for the given grantee and with given permissions - * @param grantee - * @param permission - */ - public ObjectGrant(Grantee grantee, ObjectPermissions permission) { - super(); - this.grantee = grantee; - this.permission = permission; - } - - /** - * Gets the grantee for this particular object permission - */ - public Grantee getGrantee() { - return grantee; - } - - /** - * Gets the Corresponding object permission - */ - public ObjectPermissions getPermission() { - return permission; - } - - - @Override - public int hashCode() { - final int prime = 31; - int result = 1; - result = prime * result + ((grantee == null) ? 0 : grantee.hashCode()); - result = prime * result - + ((permission == null) ? 0 : permission.hashCode()); - return result; - } - - - @Override - public boolean equals(Object obj) { - if (this == obj) - return true; - if (obj == null) - return false; - if (getClass() != obj.getClass()) - return false; - ObjectGrant other = (ObjectGrant) obj; - if (grantee == null) { - if (other.grantee != null) - return false; - } else if (!grantee.equals(other.grantee)) - return false; - if (permission != other.permission) - return false; - return true; - } -} diff --git a/src/main/java/org/springframework/integration/aws/s3/core/ObjectPermissions.java b/src/main/java/org/springframework/integration/aws/s3/core/ObjectPermissions.java deleted file mode 100644 index 5fefc68..0000000 --- a/src/main/java/org/springframework/integration/aws/s3/core/ObjectPermissions.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.aws.s3.core; - -/** - * Represents the various types of permissions on the object in S3 - * - * @author Amol Nayak - * - * @since 0.5 - * - */ -public enum ObjectPermissions { - - /** - * Indicates the grantee has permissions to read the object from the containing bucket - */ - READ, - - /** - * Indicates the grantee has permissions to read the Access control permissions of the - * Object in S3 - */ - READ_ACP, - - /** - * Indicates the grantee has permissions to write the Access control permissions of the - * Object in S3 - */ - WRITE_ACP; -} diff --git a/src/main/java/org/springframework/integration/aws/s3/core/PaginatedObjectsView.java b/src/main/java/org/springframework/integration/aws/s3/core/PaginatedObjectsView.java deleted file mode 100644 index f2aa2d7..0000000 --- a/src/main/java/org/springframework/integration/aws/s3/core/PaginatedObjectsView.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.aws.s3.core; - -import java.util.List; - -/** - * Returns the Paginated view of the objects in Amazon S3 for the queries bucket - * See {@link AmazonS3Operations} for more details on various operations on S3 - * - * @author Amol Nayak - * - * @since 0.5 - * - */ -public interface PaginatedObjectsView { - - /** - * Gets the Paginated List of Object names - * @return A {@link List} of paginated object names - */ - List getObjectSummary(); - - /** - * Invoke this method to know if more pages of results is present - * @return true if more results are present in the {@link List} of objects returned - */ - boolean hasMoreResults(); - - /** - * Contains the marker that can be used to get the next listing of objects from the - * S3. Contains a null value if the listing is complete, the hasMoreResults - * method will return true if this marker contains a non null value. - */ - String getNextMarker(); - -} diff --git a/src/main/java/org/springframework/integration/aws/s3/core/S3ObjectSummary.java b/src/main/java/org/springframework/integration/aws/s3/core/S3ObjectSummary.java deleted file mode 100644 index 3344e9a..0000000 --- a/src/main/java/org/springframework/integration/aws/s3/core/S3ObjectSummary.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright 2002-2016 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.aws.s3.core; - -import java.util.Date; - -/** - * The summary of the Object stored on Amazon S3. - * - * @author Amol Nayak - * - * @since 0.5 - * - */ -public interface S3ObjectSummary { - - /** - * Gets the Bucket name in which the object is kept on S3 - */ - String getBucketName(); - - /** - * Gets the keys under which the Object is stored on S3 - */ - String getKey(); - - /** - * Gets the Hex encoded 128 bit MD5 digest of the contents of the object uploaded on S3 - */ - String getETag(); - - /** - * Gets the size of the object in bytes - */ - long getSize(); - - /** - * Gets the Date the object was last modified - */ - Date getLastModified(); - -} diff --git a/src/main/java/org/springframework/integration/aws/support/S3RemoteFileTemplate.java b/src/main/java/org/springframework/integration/aws/support/S3RemoteFileTemplate.java new file mode 100644 index 0000000..5e4f588 --- /dev/null +++ b/src/main/java/org/springframework/integration/aws/support/S3RemoteFileTemplate.java @@ -0,0 +1,68 @@ +/* + * Copyright 2016 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.aws.support; + +import java.io.IOException; + +import org.springframework.integration.file.remote.ClientCallback; +import org.springframework.integration.file.remote.RemoteFileTemplate; +import org.springframework.integration.file.remote.session.SessionFactory; + +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.AmazonS3Exception; +import com.amazonaws.services.s3.model.S3ObjectSummary; + +/** + * An Amazon S3 specific {@link RemoteFileTemplate} extension. + * + * @author Artem Bilan + */ +public class S3RemoteFileTemplate extends RemoteFileTemplate { + + public S3RemoteFileTemplate() { + this(new S3SessionFactory()); + } + + public S3RemoteFileTemplate(AmazonS3 amazonS3) { + this(new S3SessionFactory(amazonS3)); + } + + /** + * Construct a {@link RemoteFileTemplate} with the supplied session factory. + * @param sessionFactory the session factory. + */ + public S3RemoteFileTemplate(SessionFactory sessionFactory) { + super(sessionFactory); + } + + @SuppressWarnings("unchecked") + @Override + public T executeWithClient(final ClientCallback callback) { + return callback.doWithClient((C) this.sessionFactory.getSession().getClientInstance()); + } + + @Override + public boolean exists(final String path) { + try { + return this.sessionFactory.getSession().exists(path); + } + catch (IOException e) { + throw new AmazonS3Exception("Failed to check the path " + path, e); + } + } + +} diff --git a/src/main/java/org/springframework/integration/aws/support/S3Session.java b/src/main/java/org/springframework/integration/aws/support/S3Session.java new file mode 100644 index 0000000..6849ade --- /dev/null +++ b/src/main/java/org/springframework/integration/aws/support/S3Session.java @@ -0,0 +1,216 @@ +/* + * Copyright 2002-2016 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.aws.support; + +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.util.ArrayList; +import java.util.List; + +import org.apache.http.HttpStatus; + +import org.springframework.integration.file.remote.session.Session; +import org.springframework.util.Assert; +import org.springframework.util.StreamUtils; + +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.AmazonS3Exception; +import com.amazonaws.services.s3.model.CopyObjectRequest; +import com.amazonaws.services.s3.model.ListObjectsRequest; +import com.amazonaws.services.s3.model.ObjectListing; +import com.amazonaws.services.s3.model.ObjectMetadata; +import com.amazonaws.services.s3.model.S3Object; +import com.amazonaws.services.s3.model.S3ObjectInputStream; +import com.amazonaws.services.s3.model.S3ObjectSummary; + +/** + * An Amazon S3 {@link Session} implementation. + * + * @author Artem Bilan + */ +public class S3Session implements Session { + + private final AmazonS3 amazonS3; + + public S3Session(AmazonS3 amazonS3) { + Assert.notNull(amazonS3, "'amazonS3' must not be null."); + this.amazonS3 = amazonS3; + } + + @Override + public S3ObjectSummary[] list(String path) throws IOException { + Assert.hasText(path, "'path' must not be empty String."); + String[] bucketPrefix = path.split("/"); + ListObjectsRequest listObjectsRequest = new ListObjectsRequest() + .withBucketName(bucketPrefix[0]); + if (bucketPrefix.length > 1) { + listObjectsRequest.setPrefix(bucketPrefix[1]); + } + + /* + For listing objects, Amazon S3 returns up to 1,000 keys in the response. + If you have more than 1,000 keys in your bucket, the response will be truncated. + You should always check for if the response is truncated. + */ + ObjectListing objectListing; + List objectSummaries = new ArrayList<>(); + do { + objectListing = this.amazonS3.listObjects(listObjectsRequest); + objectSummaries.addAll(objectListing.getObjectSummaries()); + listObjectsRequest.setMarker(objectListing.getNextMarker()); + } + while (objectListing.isTruncated()); + + return objectSummaries.toArray(new S3ObjectSummary[objectSummaries.size()]); + } + + @Override + public String[] listNames(String path) throws IOException { + String[] bucketPrefix = path.split("/"); + ListObjectsRequest listObjectsRequest = new ListObjectsRequest() + .withBucketName(bucketPrefix[0]); + if (bucketPrefix.length > 1) { + listObjectsRequest.setPrefix(bucketPrefix[1]); + } + + /* + For listing objects, Amazon S3 returns up to 1,000 keys in the response. + If you have more than 1,000 keys in your bucket, the response will be truncated. + You should always check for if the response is truncated. + */ + ObjectListing objectListing; + List names = new ArrayList<>(); + do { + objectListing = this.amazonS3.listObjects(listObjectsRequest); + for (S3ObjectSummary objectSummary : objectListing.getObjectSummaries()) { + names.add(objectSummary.getKey()); + } + listObjectsRequest.setMarker(objectListing.getNextMarker()); + } + while (objectListing.isTruncated()); + + return names.toArray(new String[names.size()]); + } + + @Override + public boolean remove(String path) throws IOException { + String[] bucketKey = splitPathToBucketAndKey(path); + this.amazonS3.deleteObject(bucketKey[0], bucketKey[1]); + return true; + } + + @Override + public void rename(String pathFrom, String pathTo) throws IOException { + String[] bucketKeyFrom = splitPathToBucketAndKey(pathFrom); + String[] bucketKeyTo = splitPathToBucketAndKey(pathTo); + CopyObjectRequest copyRequest = new CopyObjectRequest(bucketKeyFrom[0], bucketKeyFrom[1], + bucketKeyTo[0], bucketKeyTo[1]); + this.amazonS3.copyObject(copyRequest); + + //Delete the source + this.amazonS3.deleteObject(bucketKeyFrom[0], bucketKeyFrom[1]); + } + + @Override + public void read(String source, OutputStream outputStream) throws IOException { + String[] bucketKey = splitPathToBucketAndKey(source); + S3Object s3Object = this.amazonS3.getObject(bucketKey[0], bucketKey[1]); + S3ObjectInputStream objectContent = s3Object.getObjectContent(); + try { + StreamUtils.copy(objectContent, outputStream); + } + finally { + objectContent.close(); + } + } + + @Override + public void write(InputStream inputStream, String destination) throws IOException { + Assert.notNull(inputStream, "'inputStream' must not be null."); + String[] bucketKey = splitPathToBucketAndKey(destination); + this.amazonS3.putObject(bucketKey[0], bucketKey[1], inputStream, new ObjectMetadata()); + } + + @Override + public void append(InputStream inputStream, String destination) throws IOException { + throw new UnsupportedOperationException("The 'append' operation isn't supported by the Amazon S3 protocol."); + } + + @Override + public boolean mkdir(String directory) throws IOException { + this.amazonS3.createBucket(directory); + return true; + } + + @Override + public boolean rmdir(String directory) throws IOException { + this.amazonS3.deleteBucket(directory); + return true; + } + + @Override + public boolean exists(String path) throws IOException { + String[] bucketKey = splitPathToBucketAndKey(path); + try { + this.amazonS3.getObjectMetadata(bucketKey[0], bucketKey[1]); + } + catch (AmazonS3Exception e) { + if (HttpStatus.SC_NOT_FOUND == e.getStatusCode()) { + return false; + } + else { + throw e; + } + } + return true; + } + + @Override + public InputStream readRaw(String source) throws IOException { + String[] bucketKey = splitPathToBucketAndKey(source); + S3Object s3Object = this.amazonS3.getObject(bucketKey[0], bucketKey[1]); + return s3Object.getObjectContent(); + } + + @Override + public void close() { + // No-op. This session is just direct wrapper for the AmazonS3 + } + + @Override + public boolean isOpen() { + return true; + } + + @Override + public boolean finalizeRaw() throws IOException { + return true; + } + + @Override + public Object getClientInstance() { + return this.amazonS3; + } + + private static String[] splitPathToBucketAndKey(String path) { + Assert.hasText(path, "'path' must not be empty String."); + Assert.state(path.indexOf('/') != -1, "'path' must in pattern [BUCKET/KEY]."); + return path.split("/"); + } + +} diff --git a/src/main/java/org/springframework/integration/aws/support/S3SessionFactory.java b/src/main/java/org/springframework/integration/aws/support/S3SessionFactory.java new file mode 100644 index 0000000..26005aa --- /dev/null +++ b/src/main/java/org/springframework/integration/aws/support/S3SessionFactory.java @@ -0,0 +1,62 @@ +/* + * Copyright 2016 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.aws.support; + +import org.springframework.integration.file.remote.session.SessionFactory; +import org.springframework.integration.file.remote.session.SharedSessionCapable; +import org.springframework.util.Assert; + +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3Client; +import com.amazonaws.services.s3.model.S3ObjectSummary; + +/** + * An Amazon S3 specific {@link SessionFactory} implementation. + * Also this class implements {@link SharedSessionCapable} around the single instance, + * since the {@link S3Session} is simple thread-safe wrapper for the {@link AmazonS3}. + * + * @author Artem Bilan + */ +public class S3SessionFactory implements SessionFactory, SharedSessionCapable { + + private final S3Session s3Session; + + public S3SessionFactory() { + this(new AmazonS3Client()); + } + + public S3SessionFactory(AmazonS3 amazonS3) { + Assert.notNull(amazonS3, "'amazonS3' must not be null."); + this.s3Session = new S3Session(amazonS3); + } + + @Override + public S3Session getSession() { + return this.s3Session; + } + + @Override + public boolean isSharedSession() { + return true; + } + + @Override + public void resetSharedSession() { + // No-op. The S3Session is stateless and can be used concurrently. + } + +} diff --git a/src/main/java/org/springframework/integration/aws/support/filters/S3PersistentAcceptOnceFileListFilter.java b/src/main/java/org/springframework/integration/aws/support/filters/S3PersistentAcceptOnceFileListFilter.java new file mode 100644 index 0000000..49e072d --- /dev/null +++ b/src/main/java/org/springframework/integration/aws/support/filters/S3PersistentAcceptOnceFileListFilter.java @@ -0,0 +1,47 @@ +/* + * Copyright 2016 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.aws.support.filters; + + +import org.springframework.integration.file.filters.AbstractPersistentAcceptOnceFileListFilter; +import org.springframework.integration.metadata.ConcurrentMetadataStore; + +import com.amazonaws.services.s3.model.S3ObjectSummary; + +/** + * Persistent file list filter using the server's file timestamp to detect if we've already + * 'seen' this file. + * + * @author Artem Bilan + */ +public class S3PersistentAcceptOnceFileListFilter extends AbstractPersistentAcceptOnceFileListFilter { + + public S3PersistentAcceptOnceFileListFilter(ConcurrentMetadataStore store, String prefix) { + super(store, prefix); + } + + @Override + protected long modified(S3ObjectSummary file) { + return (file != null) ? file.getLastModified().getTime() : 0L; + } + + @Override + protected String fileName(S3ObjectSummary file) { + return (file != null) ? file.getKey() : null; + } + +} diff --git a/src/main/java/org/springframework/integration/aws/support/filters/S3RegexPatternFileListFilter.java b/src/main/java/org/springframework/integration/aws/support/filters/S3RegexPatternFileListFilter.java new file mode 100644 index 0000000..20cd8f2 --- /dev/null +++ b/src/main/java/org/springframework/integration/aws/support/filters/S3RegexPatternFileListFilter.java @@ -0,0 +1,45 @@ +/* + * Copyright 2016 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.aws.support.filters; + +import java.util.regex.Pattern; + +import org.springframework.integration.file.filters.AbstractRegexPatternFileListFilter; + +import com.amazonaws.services.s3.model.S3ObjectSummary; + +/** + * Implementation of {@link AbstractRegexPatternFileListFilter} for Amazon S3. + * + * @author Artem Bilan + */ +public class S3RegexPatternFileListFilter extends AbstractRegexPatternFileListFilter { + + public S3RegexPatternFileListFilter(String pattern) { + super(pattern); + } + + public S3RegexPatternFileListFilter(Pattern pattern) { + super(pattern); + } + + @Override + protected String getFilename(S3ObjectSummary file) { + return (file != null) ? file.getKey() : null; + } + +} diff --git a/src/main/java/org/springframework/integration/aws/support/filters/S3SimplePatternFileListFilter.java b/src/main/java/org/springframework/integration/aws/support/filters/S3SimplePatternFileListFilter.java new file mode 100644 index 0000000..b0ff6d9 --- /dev/null +++ b/src/main/java/org/springframework/integration/aws/support/filters/S3SimplePatternFileListFilter.java @@ -0,0 +1,39 @@ +/* + * Copyright 2016 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.aws.support.filters; + +import org.springframework.integration.file.filters.AbstractSimplePatternFileListFilter; + +import com.amazonaws.services.s3.model.S3ObjectSummary; + +/** + * Implementation of {@link AbstractSimplePatternFileListFilter} for Amazon S3. + * + * @author Artem Bilan + */ +public class S3SimplePatternFileListFilter extends AbstractSimplePatternFileListFilter { + + public S3SimplePatternFileListFilter(String pattern) { + super(pattern); + } + + @Override + protected String getFilename(S3ObjectSummary file) { + return (file != null) ? file.getKey() : null; + } + +} diff --git a/src/main/java/org/springframework/integration/aws/support/filters/package-info.java b/src/main/java/org/springframework/integration/aws/support/filters/package-info.java new file mode 100644 index 0000000..ff4ccef --- /dev/null +++ b/src/main/java/org/springframework/integration/aws/support/filters/package-info.java @@ -0,0 +1,4 @@ +/** + * Provides classes supporting FTP file filtering. + */ +package org.springframework.integration.aws.support.filters; diff --git a/src/main/resources/org/springframework/integration/aws/config/spring-integration-aws-1.0.xsd b/src/main/resources/org/springframework/integration/aws/config/spring-integration-aws-1.0.xsd index 0452c30..041f68f 100644 --- a/src/main/resources/org/springframework/integration/aws/config/spring-integration-aws-1.0.xsd +++ b/src/main/resources/org/springframework/integration/aws/config/spring-integration-aws-1.0.xsd @@ -288,128 +288,178 @@ - Defines the inbound channel adapter for Amazon S3. The component is used to synchronize - the objects in an S3 bucket with the file system. + Configures a 'SourcePollingChannelAdapter' Endpoint for the + 'org.springframework.integration.aws.inbound.S3InboundFileSynchronizingMessageSource' that + synchronizes a local directory with the contents of a remote Amazon S3 bucket. - - + - + + + + + + + + + + Identifies the directory path (e.g., + "/local/mytransfers") where files + will be transferred TO. + + + + + + + Identifies the remote directory path (e.g., "/remote/mytransfers") + Mutually exclusive with 'remote-directory-expression'. + + + + + + + Specify a SpEL expression which + will be used to evaluate the directory + path from where the files will be transferred + (e.g., "@someBean.fetchDirectory"); + Mutually exclusive with 'remote-directory'. + + + + + + + Extension used when downloading files. + We change it right after we know it's downloaded. + + + + + + + Allows you to provide a file name pattern to + determine the file names + that need to be scanned. + This is based on + simple pattern matching (e.g., "*.txt, fo*.txt" etc.) + + + + + + + Allows you to provide a SpEL expression to + generate the file name of + the local (transferred) file. The root + object of the SpEL + evaluation is the name of the original + file. + For example, a valid expression would be "#this.toUpperCase() + + '.a'" where #this represents the + original name of the remote + file. + + + + + + + Allows you to provide a Regular Expression to + determine the file names + that need to be scanned. + (e.g., "f[o]+\.txt" etc.) + + + + + + + Specify a Comparator to be used when ordering Files. If none is provided, the + order will be determined by the java.io.File implementation of Comparable. + + + + - - - - - The mandatory attribute that would be used to provide the AWS bucket to which - the objects needs to be uploaded. - - - - - - - The suffix files will use while the are being written to the local file system. - A file present with this suffix on the local file system denotes that the file - is not completely received from the s3 bucket. - - - - + - + + + - Reference to the bean with an implementation of - org.springframework.integration.aws.s3.core.AmazonS3Operations - that would be used to perform the operations on the S3 bucket. If not provided, the - default implementation used is - org.springframework.integration.aws.s3.core.DefaultAmazonS3Operations which uses AWS SDK. + Allows you to specify a reference to a + [org.springframework.integration.file.filters.FileListFilter] + bean. This filter is applied to files on the remote server and + only files that pass the filter are retrieved. - + + + + + + + + + Allows you to specify a reference to a + [org.springframework.integration.file.filters.FileListFilter] + bean. This filter is applied to files after they have been + retrieved. The default is an AcceptOnceFileListFilter which means that, + even if a new instance of a file is retrieved from the remote server, + a message won't be generated. The filter provided here is combined + with a filter that prevents the message source from processing + files that are currently being downloaded. + + + + - The String that gives the endpoint to use for the adapter, if none is - specified the default used is s3.amazonaws.com. + Tells this adapter if the local directory must + be auto-created if it + doesn't exist. Default is TRUE. - + - This is the sub folder if any on the remote bucket that would be synchronized - with the local directory. Useful if a part of the bucket is to be synchronized. - If none specified, the entire bucket will be synchronized with the local directory. + Specify whether to delete the remote source + file after copying. + By default, the remote files will NOT be + deleted. - + - The attribute specifying the directory on the local file system where - the objects from S3 bucket would be synchronized to. Either of local-directory - or the local-directory-expression are mandatory. - - - - - - - The attribute specifying the expression to find the directory on the local file - system where the objects from S3 bucket would be synchronized to. Either of local-directory - or the locl-directory-expression are mandatory. - - - - - - - - The maximum number of objects returned in the listOperation performed on the S3 bucket - The default value used internally is 100. That is not more than 100 objects would be returned - in one call to list the objects, subsequent calls will be made to the Web service to retrieve - the next - batch of objects. - - - - - - - The boolean value that would be used to specify if the sub folders of the given - folder are to be synchronized or not. By default the value is false and only files - at the level of the specified folder (or root of the bucket if no remote folder specified) - are synchronized. The objects in the sub folder are ignored. - - - - - - - The wildcard pattern that would be used to further filter out the objects listed. - This attribute is mutually exclusive to the file-name-regex attribute. - - - - - - - The regex that would be used to further filter out the objects listed. - This attribute is mutually exclusive to the file-name-wildcard attribute. + Specify whether to preserve the modified timestamp from the remote source + file on the local file after copying. + By default, the remote timestamp will NOT be + preserved. + diff --git a/src/test/java/org/springframework/integration/aws/common/AWSClientFactoryTest.java b/src/test/java/org/springframework/integration/aws/common/AWSClientFactoryTest.java deleted file mode 100644 index a52dee0..0000000 --- a/src/test/java/org/springframework/integration/aws/common/AWSClientFactoryTest.java +++ /dev/null @@ -1,60 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.aws.common; - -import org.junit.Assert; - -import org.junit.Test; -import org.springframework.integration.aws.core.AbstractAWSClientFactory; - -import com.amazonaws.auth.AWSCredentials; -import com.amazonaws.services.sqs.AmazonSQSClient; - -/** - * The test class for {@link AbstractAmazonWSClientFactory} - * - * @author Amol Nayak - * - * @since 0.5 - * - */ -public class AWSClientFactoryTest { - - AbstractAWSClientFactory factory = new AbstractAWSClientFactory() { - - @Override - protected AmazonSQSClient getClientImplementation() { - return new AmazonSQSClient((AWSCredentials)null); - } - - }; - - @Test - public void getRegionSpecificEndpoints() { - //TODO: Use Spring integration test to inspect and assert the variables within - AmazonSQSClient usEastClient = factory.getClient("https://queue.amazonaws.com/123456789012/MyTestQueue"); - Assert.assertNotNull(usEastClient); - Assert.assertEquals(factory.getClientMap().size(),1); - AmazonSQSClient usEastClient1 = factory.getClient("https://queue.amazonaws.com/123456789012/MyTestQueue1"); - Assert.assertNotNull(usEastClient1); - Assert.assertEquals(usEastClient,usEastClient1); - Assert.assertEquals(factory.getClientMap().size(),1); - AmazonSQSClient apacClient1 = factory.getClient("https://ap-southeast-1.queue.amazonaws.com/123456789012/MyApacTestQueue"); - Assert.assertNotNull(apacClient1); - Assert.assertEquals(factory.getClientMap().size(),2); - - } -} diff --git a/src/test/java/org/springframework/integration/aws/common/AWSTestUtils.java b/src/test/java/org/springframework/integration/aws/common/AWSTestUtils.java deleted file mode 100644 index c2e145c..0000000 --- a/src/test/java/org/springframework/integration/aws/common/AWSTestUtils.java +++ /dev/null @@ -1,165 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.aws.common; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStreamReader; -import java.io.UnsupportedEncodingException; -import java.security.MessageDigest; -import java.security.NoSuchAlgorithmException; -import java.util.ArrayList; -import java.util.List; - -import org.junit.Assert; - -import org.apache.commons.codec.binary.Base64; -import org.springframework.integration.aws.core.AWSCredentials; -import org.springframework.integration.aws.core.PropertiesAWSCredentials; - -/** - * Common test class utility to be used by AmazonWS test cases - * - * @author Amol Nayak - * - * @since 0.5 - * - */ -public final class AWSTestUtils { - - private static PropertiesAWSCredentials credentials; - - private AWSTestUtils() { - throw new AssertionError("Cannot instantiate a utility class"); - } - - /** - * Method that will be used to test the contents of the file to assert we are getting the - * the right value - * - * @param permFile - * @param expectedContent - * @throws IOException - */ - public static final void assertFileContent(File permFile, String expectedContent) - throws IOException { - BufferedReader reader = null; - try { - reader = new BufferedReader(new InputStreamReader(new FileInputStream(permFile))); - Assert.assertEquals(expectedContent, reader.readLine()); - } finally { - if(reader != null) { - reader.close(); - } - } - } - - /** - * Writes the content to the file at given location. - * - * @param path - * @param content - */ - public static final void writeToFile(String path, String content) throws IOException { - //create the required directories if needed - if(path.contains(File.separator)) { - int index = path.lastIndexOf(File.separatorChar); - if(index != 0) { - new File(path.substring(0, index)).mkdirs(); - } - } - File file = new File(path); - FileOutputStream fos = new FileOutputStream(file); - fos.write(content.getBytes()); - fos.close(); - } - - /** - * The static helper method that would be used by other AWS tests to - * get the implementation of the {@link AWSCredentials} instance - * @throws Exception - */ - public static AWSCredentials getCredentials() { - if(credentials == null) { - credentials = - new PropertiesAWSCredentials("classpath:awscredentials.properties"); - try { - credentials.afterPropertiesSet(); - } catch (Exception e) { - return null; - } - } - return credentials; - } - - /** - * When passed a {@link File} instance for the root directory, the contents are recursively - * checked and the listing of the directories is retrieved. - * - * @param rootDirectory - */ - public static List getContentsRecursively(File rootDirectory) { - if(!rootDirectory.isDirectory()) { - return null; - } - else { - List files = new ArrayList(); - getContentsRecursively(rootDirectory, files); - return files; - } - } - - /** - * Recursively gets all the files present under the provided directory - * @param file - * @param files - */ - private static void getContentsRecursively(File file, List files) { - if(file.isFile()) { - files.add(file); - } - else if(file.isDirectory()){ - File[] children = file.listFiles(); - if(children != null && children.length != 0) { - for(File child:children) { - getContentsRecursively(child, files); - } - } - } - } - - /** - * Helper method that will be used to generate the base64 encoded MD5 hash of the string - * - * @param input - */ - public static String md5Hash(String input) { - try { - MessageDigest digest = MessageDigest.getInstance("MD5"); - byte[] digestedBytes = digest.digest(input.getBytes("UTF-8")); - return new String(Base64.encodeBase64(digestedBytes),"UTF-8"); - } catch (NoSuchAlgorithmException e) { - e.printStackTrace(); - return null; - } catch(UnsupportedEncodingException e) { - e.printStackTrace(); - return null; - } - } -} \ No newline at end of file diff --git a/src/test/java/org/springframework/integration/aws/common/BaseTestCase.java b/src/test/java/org/springframework/integration/aws/common/BaseTestCase.java deleted file mode 100644 index 74dc96e..0000000 --- a/src/test/java/org/springframework/integration/aws/common/BaseTestCase.java +++ /dev/null @@ -1,51 +0,0 @@ -/* - * Copyright 2002-2012 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.aws.common; - -import java.io.IOException; -import java.util.Properties; - -/** - * Class extended by some of the test cases which are not spring based to get access to the properties - * and other common functionality - * - * @author Amol Nayak - * - * @since 0.5 - * - */ -public abstract class BaseTestCase { - - protected static String propsLocation = "testprops.properties"; - protected static Properties props; - static { - props = new Properties(); - try { - props.load(ClassLoader.getSystemClassLoader().getResourceAsStream(propsLocation)); - } catch (IOException e) { - // TODO Auto-generated catch block - e.printStackTrace(); - } - } - - protected static String getProperty(String key) { - if(props != null) - return props.getProperty(key); - else - return null; - } - -} diff --git a/src/test/java/org/springframework/integration/aws/config/xml/AbstractAWSOutboundChannelAdapterParserTests.java b/src/test/java/org/springframework/integration/aws/config/xml/AbstractAWSOutboundChannelAdapterParserTests.java deleted file mode 100644 index 1435e91..0000000 --- a/src/test/java/org/springframework/integration/aws/config/xml/AbstractAWSOutboundChannelAdapterParserTests.java +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Copyright 2002-2014 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.aws.config.xml; - -import org.junit.Assert; - -import org.junit.AfterClass; -import org.junit.Before; -import org.junit.Test; -import org.springframework.context.ApplicationContext; -import org.springframework.context.support.ClassPathXmlApplicationContext; -import org.springframework.integration.aws.core.AWSCredentials; -import org.springframework.integration.endpoint.AbstractEndpoint; -import org.springframework.integration.test.util.TestUtils; -import org.springframework.messaging.MessageHandler; - -/** - * The Abstract test class for the AWS outbound adapter parsers - * - * @author Amol Nayak - * @author Rob Harrop - * - * @since 0.5 - * - */ -public abstract class AbstractAWSOutboundChannelAdapterParserTests { - - protected static ClassPathXmlApplicationContext ctx; - private static boolean isInitialized; - - /** - * The bean name expected to be present in the context which would be used when the - * element is defined with the propertiesFile attribute - */ - private final String CREDENTIALS_TEST_ONE = "credentialsTestOne"; - - /** - * The bean name expected to be present in the context which would be used when the - * element is defined with the accessKey and the secretKey definition - */ - private final String CREDENTIALS_TEST_TWO = "credentialsTestTwo"; - - - @Before - public void setup() { - if(!isInitialized) { - String contextLocation = getConfigFilePath(); - Assert.assertNotNull("Non null path value expected", contextLocation); - ctx = new ClassPathXmlApplicationContext(contextLocation); - Assert.assertTrue("Bean with id " + CREDENTIALS_TEST_ONE - + " expected to be present in the context for credentials test", - ctx.containsBean(CREDENTIALS_TEST_ONE)); - Assert.assertTrue("Bean with id " + CREDENTIALS_TEST_TWO - + " expected to be present in the context for credentials test", - ctx.containsBean(CREDENTIALS_TEST_TWO)); - isInitialized = true; - - } - } - - @AfterClass - public static void destroy() { - ctx.close(); - } - - /** - * Gets the Message handler implementation for the given bean id - * @param beanId - */ - @SuppressWarnings("unchecked") - protected T getMessageHandlerForBeanDefinition(String beanId) { - AbstractEndpoint endpoint = (AbstractEndpoint)ctx.getBean(beanId); - return (T)TestUtils.getPropertyValue(endpoint, "handler"); - } - - - /** - * Gets the config file path that would be used to create the {@link ApplicationContext} instance - * sub class should implement this method to return a value of the config to be used to create the - * context - * - */ - protected abstract String getConfigFilePath(); - - /** - * The subclass should return the {@link AWSCredentials} instance that is being used - * to configure the adapters - * - */ - protected abstract AWSCredentials getCredentials(); - - /** - * The test class for the AWS Credentials test that used property files - */ - @Test - public final void awsCredentialsTestWithPropFiles() { - MessageHandler handler = getMessageHandlerForBeanDefinition(CREDENTIALS_TEST_ONE); - AWSCredentials credentials = getCredentials(); - String accessKey = credentials.getAccessKey(); - String secretKey = credentials.getSecretKey(); - AWSCredentials configuredCredentials = - TestUtils.getPropertyValue(handler, "credentials",AWSCredentials.class); - Assert.assertEquals(accessKey, configuredCredentials.getAccessKey()); - Assert.assertEquals(secretKey, configuredCredentials.getSecretKey()); - } - - /** - * The test class for the AWS Credentials test that used accessKey and secretKey elements - */ - @Test - public final void awsCredentialsTestWithoutPropFiles() { - MessageHandler handler = getMessageHandlerForBeanDefinition(CREDENTIALS_TEST_TWO); - AWSCredentials credentials = getCredentials(); - String accessKey = credentials.getAccessKey(); - String secretKey = credentials.getSecretKey(); - AWSCredentials configuredCredentials = - TestUtils.getPropertyValue(handler, "credentials",AWSCredentials.class); - Assert.assertEquals(accessKey, configuredCredentials.getAccessKey()); - Assert.assertEquals(secretKey, configuredCredentials.getSecretKey()); - } -} diff --git a/src/test/java/org/springframework/integration/aws/config/xml/S3InboundChannelAdapterParserTests-context.xml b/src/test/java/org/springframework/integration/aws/config/xml/S3InboundChannelAdapterParserTests-context.xml new file mode 100644 index 0000000..045e002 --- /dev/null +++ b/src/test/java/org/springframework/integration/aws/config/xml/S3InboundChannelAdapterParserTests-context.xml @@ -0,0 +1,49 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/test/java/org/springframework/integration/aws/config/xml/S3InboundChannelAdapterParserTests.java b/src/test/java/org/springframework/integration/aws/config/xml/S3InboundChannelAdapterParserTests.java new file mode 100644 index 0000000..0cb7a14 --- /dev/null +++ b/src/test/java/org/springframework/integration/aws/config/xml/S3InboundChannelAdapterParserTests.java @@ -0,0 +1,117 @@ +/* + * Copyright 2016 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.aws.config.xml; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertSame; +import static org.junit.Assert.assertTrue; + +import java.lang.reflect.Method; +import java.util.Collection; +import java.util.Comparator; +import java.util.concurrent.PriorityBlockingQueue; +import java.util.concurrent.atomic.AtomicReference; + +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.expression.Expression; +import org.springframework.integration.aws.inbound.S3InboundFileSynchronizer; +import org.springframework.integration.aws.inbound.S3InboundFileSynchronizingMessageSource; +import org.springframework.integration.aws.support.filters.S3SimplePatternFileListFilter; +import org.springframework.integration.endpoint.SourcePollingChannelAdapter; +import org.springframework.integration.file.filters.AcceptAllFileListFilter; +import org.springframework.integration.file.remote.session.SessionFactory; +import org.springframework.integration.file.remote.synchronizer.AbstractInboundFileSynchronizer; +import org.springframework.integration.test.util.TestUtils; +import org.springframework.messaging.MessageChannel; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import org.springframework.util.ReflectionUtils; + +/** + * @author Artem Bilan + */ +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration +public class S3InboundChannelAdapterParserTests { + + @Autowired + private SourcePollingChannelAdapter s3Inbound; + + @Autowired + private Comparator comparator; + + @Autowired + private MessageChannel s3Channel; + + @Autowired + private AcceptAllFileListFilter acceptAllFilter; + + @Autowired + private SessionFactory s3SessionFactory; + + @Test + public void testFtpInboundChannelAdapterComplete() throws Exception { + assertFalse(TestUtils.getPropertyValue(this.s3Inbound, "autoStartup", Boolean.class)); + PriorityBlockingQueue blockingQueue = TestUtils.getPropertyValue(this.s3Inbound, + "source.fileSource.toBeReceived", PriorityBlockingQueue.class); + Comparator comparator = blockingQueue.comparator(); + assertSame(this.comparator, comparator); + assertEquals("s3Inbound", this.s3Inbound.getComponentName()); + assertEquals("aws:s3-inbound-channel-adapter", this.s3Inbound.getComponentType()); + assertSame(this.s3Channel, TestUtils.getPropertyValue(this.s3Inbound, "outputChannel")); + + S3InboundFileSynchronizingMessageSource inbound = TestUtils.getPropertyValue(this.s3Inbound, "source", + S3InboundFileSynchronizingMessageSource.class); + + S3InboundFileSynchronizer fisync = TestUtils.getPropertyValue(inbound, "synchronizer", + S3InboundFileSynchronizer.class); + assertEquals("'foo/bar'", + TestUtils.getPropertyValue(fisync, "remoteDirectoryExpression", Expression.class) + .getExpressionString()); + assertNotNull(TestUtils.getPropertyValue(fisync, "localFilenameGeneratorExpression")); + assertTrue(TestUtils.getPropertyValue(fisync, "preserveTimestamp", Boolean.class)); + assertEquals(".foo", TestUtils.getPropertyValue(fisync, "temporaryFileSuffix", String.class)); + String remoteFileSeparator = (String) TestUtils.getPropertyValue(fisync, "remoteFileSeparator"); + assertNotNull(remoteFileSeparator); + assertEquals("", remoteFileSeparator); + S3SimplePatternFileListFilter filter = TestUtils.getPropertyValue(fisync, "filter", + S3SimplePatternFileListFilter.class); + assertNotNull(filter); + assertSame(this.s3SessionFactory, TestUtils.getPropertyValue(fisync, "remoteFileTemplate.sessionFactory")); + assertTrue(TestUtils.getPropertyValue(inbound, "fileSource.scanner.filter.fileFilters", Collection.class) + .contains(this.acceptAllFilter)); + final AtomicReference genMethod = new AtomicReference(); + ReflectionUtils.doWithMethods(AbstractInboundFileSynchronizer.class, new ReflectionUtils.MethodCallback() { + + @Override + public void doWith(Method method) throws IllegalArgumentException, IllegalAccessException { + if ("generateLocalFileName".equals(method.getName())) { + method.setAccessible(true); + genMethod.set(method); + } + } + + }); + assertEquals("FOO.afoo", genMethod.get().invoke(fisync, "foo")); + } + +} diff --git a/src/test/java/org/springframework/integration/aws/core/AbstractAWSClientFactoryTests.java b/src/test/java/org/springframework/integration/aws/core/AbstractAWSClientFactoryTests.java deleted file mode 100644 index c5b9f76..0000000 --- a/src/test/java/org/springframework/integration/aws/core/AbstractAWSClientFactoryTests.java +++ /dev/null @@ -1,152 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.aws.core; - -import java.net.URI; -import java.util.Map; - -import org.junit.Assert; - -import org.junit.Before; -import org.junit.Test; -import org.springframework.integration.test.util.TestUtils; - -import com.amazonaws.AmazonWebServiceClient; - -/** - * The abstract test class for Amazon WebService client factory tests - * @author Amol Nayak - * - * @since 0.5 - * - */ -public abstract class AbstractAWSClientFactoryTests { - - protected AbstractAWSClientFactory factory; - - @Before - public final void setup() { - factory = getFactory(); - } - - /** - * The subclass is responsible for returning the appropriate factory implementation - */ - protected abstract AbstractAWSClientFactory getFactory(); - - /** - * Gets the endpoint for the service in US-EAST-1 region - */ - protected abstract String getUSEast1Endpoint(); - - - /** - * Gets the endpoint for the service in US-EAST-1 region - */ - protected abstract String getEUWest1Endpoint(); - - /** - * Gets the Suffix for the endpoint URL which is service specific - */ - protected abstract String getSuffix(); - - /** - * The test case for giving a US east endpoint without protocol - */ - @Test - public void withUSEast1EndpointWithoutProtocol() { - String usEast1 = getUSEast1Endpoint(); - factory.clear(); - T client = factory.getClient(usEast1 + getSuffix()); - Map map = factory.getClientMap(); - Assert.assertNotNull(map); - Assert.assertEquals(1, map.size()); - Assert.assertTrue("Expected one key with value https://" + usEast1, map.containsKey("https://" + usEast1)); - URI endpoint = TestUtils.getPropertyValue(client, "endpoint", URI.class); - Assert.assertNotNull(endpoint); - Assert.assertEquals(usEast1,endpoint.getHost()); - Assert.assertEquals("https",endpoint.getScheme()); - } - - /** - * Tests the factory by providing an endpoint in US east with protocol as http - */ - @Test - public void withUSEast1EndpointWithProtocol() { - String usEast1 = getUSEast1Endpoint(); - factory.clear(); - T client = factory.getClient("http://" + usEast1 + getSuffix()); - Map map = factory.getClientMap(); - Assert.assertNotNull(map); - Assert.assertEquals(1, map.size()); - Assert.assertTrue("Expected one key with value http://" + usEast1, map.containsKey("http://" + usEast1)); - URI endpoint = TestUtils.getPropertyValue(client, "endpoint", URI.class); - Assert.assertNotNull(endpoint); - Assert.assertEquals(usEast1,endpoint.getHost()); - Assert.assertEquals("http",endpoint.getScheme()); - } - - - /** - * Calls the getClient multiple times to get the same client instance on each invocation - */ - @Test - public void withMultipleCallsToSameEndpoint() { - String usEast1 = getUSEast1Endpoint(); - factory.clear(); - T client = factory.getClient("https://" + usEast1 + getSuffix()); - Map map = factory.getClientMap(); - Assert.assertNotNull(map); - Assert.assertEquals(1, map.size()); - Assert.assertTrue("Expected one key with value http://" + usEast1, map.containsKey("https://" + usEast1)); - //default to https - T client1 = factory.getClient(usEast1 + getSuffix()); - map = factory.getClientMap(); - Assert.assertEquals(1, map.size()); - Assert.assertTrue("Expected one key with value http://" + usEast1, map.containsKey("https://" + usEast1)); - Assert.assertTrue("Expecting to get the same instance of the client, but was not", client == client1); - } - - - /** - *Calls to different endpoints on the same client, expected to return a client with - *appropriate endpoint URI set - */ - @Test - public void withMultipleCallsToDifferentEndpoints() { - String usEast1 = getUSEast1Endpoint(); - String euWest1 = getEUWest1Endpoint(); - factory.clear(); - T client = factory.getClient(usEast1 + getSuffix()); - Map map = factory.getClientMap(); - Assert.assertNotNull(map); - Assert.assertEquals(1, map.size()); - Assert.assertTrue("Expected one key with value https://" + usEast1, map.containsKey("https://" + usEast1)); - URI endpoint = TestUtils.getPropertyValue(client, "endpoint", URI.class); - Assert.assertNotNull(endpoint); - Assert.assertEquals(usEast1,endpoint.getHost()); - Assert.assertEquals("https",endpoint.getScheme()); - client = factory.getClient(euWest1 + getSuffix()); - map = factory.getClientMap(); - Assert.assertNotNull(map); - Assert.assertEquals(2, map.size()); - Assert.assertTrue("Expected one key with value https://" + euWest1, map.containsKey("https://" + euWest1)); - endpoint = TestUtils.getPropertyValue(client, "endpoint", URI.class); - Assert.assertNotNull(endpoint); - Assert.assertEquals(euWest1,endpoint.getHost()); - Assert.assertEquals("https",endpoint.getScheme()); - } -} diff --git a/src/test/java/org/springframework/integration/aws/inbound/S3InboundChannelAdapterTests.java b/src/test/java/org/springframework/integration/aws/inbound/S3InboundChannelAdapterTests.java new file mode 100644 index 0000000..6c97feb --- /dev/null +++ b/src/test/java/org/springframework/integration/aws/inbound/S3InboundChannelAdapterTests.java @@ -0,0 +1,225 @@ +/* + * Copyright 2016 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.integration.aws.inbound; + +import static org.hamcrest.Matchers.instanceOf; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertThat; +import static org.junit.Assert.assertTrue; +import static org.mockito.BDDMockito.willAnswer; +import static org.mockito.Matchers.any; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileReader; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.List; + +import org.hamcrest.Matchers; +import org.junit.BeforeClass; +import org.junit.ClassRule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; +import org.mockito.Mockito; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.expression.Expression; +import org.springframework.expression.ExpressionParser; +import org.springframework.expression.spel.standard.SpelExpressionParser; +import org.springframework.integration.annotation.InboundChannelAdapter; +import org.springframework.integration.annotation.Poller; +import org.springframework.integration.aws.support.filters.S3RegexPatternFileListFilter; +import org.springframework.integration.channel.QueueChannel; +import org.springframework.integration.config.EnableIntegration; +import org.springframework.integration.file.filters.AcceptOnceFileListFilter; +import org.springframework.messaging.Message; +import org.springframework.messaging.PollableChannel; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.ContextConfiguration; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import org.springframework.util.FileCopyUtils; + +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.model.ListObjectsRequest; +import com.amazonaws.services.s3.model.ObjectListing; +import com.amazonaws.services.s3.model.S3Object; +import com.amazonaws.services.s3.model.S3ObjectSummary; + +/** + * @author Artem Bilan + */ +@RunWith(SpringJUnit4ClassRunner.class) +@ContextConfiguration +@DirtiesContext +public class S3InboundChannelAdapterTests { + + private static final ExpressionParser PARSER = new SpelExpressionParser(); + + private static final String S3_BUCKET = "S3_BUCKET"; + + @ClassRule + public static final TemporaryFolder TEMPORARY_FOLDER = new TemporaryFolder(); + + private static File REMOTE_FOLDER; + + private static List S3_OBJECTS; + + private static File LOCAL_FOLDER; + + @Autowired + private PollableChannel s3FilesChannel; + + @BeforeClass + public static void setup() throws IOException { + REMOTE_FOLDER = TEMPORARY_FOLDER.newFolder("remote"); + File aFile = new File(REMOTE_FOLDER, "a.test"); + FileCopyUtils.copy("Hello".getBytes(), aFile); + File bFile = new File(REMOTE_FOLDER, "b.test"); + FileCopyUtils.copy("Bye".getBytes(), bFile); + File otherFile = new File(REMOTE_FOLDER, "otherFile"); + FileCopyUtils.copy("Other".getBytes(), otherFile); + + S3_OBJECTS = new ArrayList<>(); + + for (File file : REMOTE_FOLDER.listFiles()) { + S3Object s3Object = new S3Object(); + s3Object.setBucketName(S3_BUCKET); + s3Object.setKey(file.getName()); + s3Object.setObjectContent(new FileInputStream(file)); + S3_OBJECTS.add(s3Object); + } + + LOCAL_FOLDER = new File(TEMPORARY_FOLDER.getRoot(), "local"); + } + + @Test + public void testS3InboundChannelAdapter() throws IOException { + Message message = this.s3FilesChannel.receive(10000); + assertNotNull(message); + assertThat(message.getPayload(), instanceOf(File.class)); + File localFile = (File) message.getPayload(); + assertEquals("A.TEST.a", localFile.getName()); + + // The test remote files are created with the current timestamp + 1 day. + assertThat(localFile.lastModified(), Matchers.greaterThan(System.currentTimeMillis())); + + message = this.s3FilesChannel.receive(10000); + assertNotNull(message); + assertThat(message.getPayload(), instanceOf(File.class)); + localFile = (File) message.getPayload(); + assertEquals("B.TEST.a", localFile.getName()); + + assertThat(localFile.lastModified(), Matchers.greaterThan(System.currentTimeMillis())); + + assertNull(this.s3FilesChannel.receive(10)); + + File file = new File(LOCAL_FOLDER, "A.TEST.a"); + assertTrue(file.exists()); + String content = FileCopyUtils.copyToString(new FileReader(file)); + assertEquals("Hello", content); + + file = new File(LOCAL_FOLDER, "B.TEST.a"); + assertTrue(file.exists()); + content = FileCopyUtils.copyToString(new FileReader(file)); + assertEquals("Bye", content); + + assertFalse(new File(LOCAL_FOLDER, "otherFile.a").exists()); + } + + @Configuration + @EnableIntegration + public static class Config { + + @Bean + public AmazonS3 amazonS3() { + AmazonS3 amazonS3 = Mockito.mock(AmazonS3.class); + + willAnswer(new Answer() { + + @Override + public ObjectListing answer(InvocationOnMock invocation) throws Throwable { + ObjectListing objectListing = new ObjectListing(); + List objectSummaries = objectListing.getObjectSummaries(); + for (S3Object s3Object : S3_OBJECTS) { + S3ObjectSummary s3ObjectSummary = new S3ObjectSummary(); + s3ObjectSummary.setBucketName(S3_BUCKET); + s3ObjectSummary.setKey(s3Object.getKey()); + Calendar calendar = Calendar.getInstance(); + calendar.add(Calendar.DATE, 1); + s3ObjectSummary.setLastModified(calendar.getTime()); + objectSummaries.add(s3ObjectSummary); + } + return objectListing; + } + + }).given(amazonS3).listObjects(any(ListObjectsRequest.class)); + + for (final S3Object s3Object : S3_OBJECTS) { + willAnswer(new Answer() { + + @Override + public S3Object answer(InvocationOnMock invocation) throws Throwable { + return s3Object; + } + + }).given(amazonS3).getObject(S3_BUCKET, s3Object.getKey()); + } + + return amazonS3; + } + + @Bean + public S3InboundFileSynchronizer s3InboundFileSynchronizer() { + S3InboundFileSynchronizer synchronizer = new S3InboundFileSynchronizer(amazonS3()); + synchronizer.setDeleteRemoteFiles(true); + synchronizer.setPreserveTimestamp(true); + synchronizer.setRemoteDirectory(S3_BUCKET); + synchronizer.setFilter(new S3RegexPatternFileListFilter(".*\\.test$")); + Expression expression = PARSER.parseExpression("#this.toUpperCase() + '.a'"); + synchronizer.setLocalFilenameGeneratorExpression(expression); + return synchronizer; + } + + @Bean + @InboundChannelAdapter(value = "s3FilesChannel", poller = @Poller(fixedDelay = "100")) + public S3InboundFileSynchronizingMessageSource s3InboundFileSynchronizingMessageSource() { + S3InboundFileSynchronizingMessageSource messageSource = + new S3InboundFileSynchronizingMessageSource(s3InboundFileSynchronizer()); + messageSource.setAutoCreateLocalDirectory(true); + messageSource.setLocalDirectory(LOCAL_FOLDER); + messageSource.setLocalFilter(new AcceptOnceFileListFilter()); + return messageSource; + } + + @Bean + public PollableChannel s3FilesChannel() { + return new QueueChannel(); + } + + } + +} diff --git a/src/test/java/org/springframework/integration/aws/s3/AmazonS3InboundSynchronizationMessageSourceTests.java b/src/test/java/org/springframework/integration/aws/s3/AmazonS3InboundSynchronizationMessageSourceTests.java deleted file mode 100644 index 4201d42..0000000 --- a/src/test/java/org/springframework/integration/aws/s3/AmazonS3InboundSynchronizationMessageSourceTests.java +++ /dev/null @@ -1,273 +0,0 @@ -/* - * Copyright 2002-2014 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.aws.s3; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNull; -import static org.springframework.integration.aws.common.AWSTestUtils.md5Hash; -import static org.springframework.integration.aws.s3.AmazonS3OperationsMockingUtil.mockAmazonS3Operations; -import static org.springframework.integration.aws.s3.AmazonS3OperationsMockingUtil.mockS3Operations; -import static org.springframework.integration.test.util.TestUtils.getPropertyValue; - -import java.io.File; -import java.io.IOException; -import java.util.Arrays; -import java.util.Map; - -import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.mockito.Mockito; - -import org.springframework.beans.factory.BeanFactory; -import org.springframework.expression.common.LiteralExpression; -import org.springframework.integration.aws.core.BasicAWSCredentials; -import org.springframework.integration.aws.s3.core.AbstractAmazonS3Operations; -import org.springframework.integration.aws.s3.core.AmazonS3Object; -import org.springframework.integration.aws.s3.core.AmazonS3ObjectACL; -import org.springframework.integration.aws.s3.core.AmazonS3Operations; -import org.springframework.integration.aws.s3.core.PaginatedObjectsView; -import org.springframework.messaging.Message; - -/** - * The test class for {@link AmazonS3InboundSynchronizationMessageSource} - * - * @author Amol Nayak - * @author Rob Harrop - * - * @since 0.5 - * - */ -public class AmazonS3InboundSynchronizationMessageSourceTests { - - @Rule - public TemporaryFolder temp = new TemporaryFolder(); - - private static AmazonS3Operations operations; - - - @BeforeClass - public static void setup() { - operations = mockS3Operations(); - } - - - /** - * Tests by providing null credentials. - */ - @Test(expected = IllegalArgumentException.class) - public void withNullCredentials() { - AmazonS3InboundSynchronizationMessageSource src = new AmazonS3InboundSynchronizationMessageSource(); - src.setCredentials(null); - } - - /** - * Tests by providing null temporary suffix - */ - @Test(expected = IllegalArgumentException.class) - public void withNullTempSuffix() { - AmazonS3InboundSynchronizationMessageSource src = new AmazonS3InboundSynchronizationMessageSource(); - src.setTemporarySuffix(null); - } - - /** - * Tests by providing null wildcard - */ - @Test(expected = IllegalArgumentException.class) - public void withNullWildcard() { - AmazonS3InboundSynchronizationMessageSource src = new AmazonS3InboundSynchronizationMessageSource(); - src.setFileNameWildcard(null); - } - - /** - * Tests by providing null regex - */ - @Test(expected = IllegalArgumentException.class) - public void withNullRegex() { - AmazonS3InboundSynchronizationMessageSource src = new AmazonS3InboundSynchronizationMessageSource(); - src.setFileNameRegex(null); - } - - /** - * Tests by providing both regex and wildcard - */ - @Test(expected = IllegalArgumentException.class) - public void withBothRegexAndWildcard() { - AmazonS3InboundSynchronizationMessageSource src = new AmazonS3InboundSynchronizationMessageSource(); - src.setFileNameRegex("[a-z]+\\.txt"); - src.setFileNameWildcard("*.txt"); - } - - /** - * Tests by providing both wildcard and regex, unlike previous one, this sets the wildcard first - */ - @Test(expected = IllegalArgumentException.class) - public void withBothWildcardAndRegex() { - AmazonS3InboundSynchronizationMessageSource src = new AmazonS3InboundSynchronizationMessageSource(); - src.setFileNameWildcard("*.txt"); - src.setFileNameRegex("[a-z]+\\.txt"); - } - - - /** - * Tests providing null remote directory - */ - @Test(expected = IllegalArgumentException.class) - public void withNullRemoteDirectory() { - AmazonS3InboundSynchronizationMessageSource src = new AmazonS3InboundSynchronizationMessageSource(); - src.setRemoteDirectory(null); - } - - /** - *Tests with a non existent local directory - */ - @Test(expected = IllegalArgumentException.class) - public void withNonExistentLocalDirectory() { - AmazonS3InboundSynchronizationMessageSource src = new AmazonS3InboundSynchronizationMessageSource(); - src.setDirectory(new LiteralExpression("SomeNotExistentDir")); - src.setBeanFactory(Mockito.mock(BeanFactory.class)); - src.afterPropertiesSet(); - } - - /** - * Tests with a {@link File} instance that is not a directory - * @throws IOException - */ - @Test(expected = IllegalArgumentException.class) - public void withNonDirectory() throws IOException { - File file = temp.newFile("SomeFile.txt"); - AmazonS3InboundSynchronizationMessageSource src = new AmazonS3InboundSynchronizationMessageSource(); - src.setDirectory(new LiteralExpression(file.getAbsolutePath())); - src.setBeanFactory(Mockito.mock(BeanFactory.class)); - src.afterPropertiesSet(); - } - - /** - * Tests with a null s3 operation. - */ - @Test(expected = IllegalArgumentException.class) - public void withNullS3Operations() { - AmazonS3InboundSynchronizationMessageSource src = new AmazonS3InboundSynchronizationMessageSource(); - src.setS3Operations(null); - } - - /** - * Doesn't set the {@link AmazonS3Operations} instance and relies on the default one. - * sets the temp suffix and the thread pool executor. - */ - @Test - public void withDefaultS3Service() { - AmazonS3InboundSynchronizationMessageSource src = new AmazonS3InboundSynchronizationMessageSource(); - BasicAWSCredentials credentials = new BasicAWSCredentials("dummy", "dummy"); - src.setTemporarySuffix(".temp"); - src.setCredentials(credentials); - src.setDirectory(new LiteralExpression(temp.getRoot().getAbsolutePath())); - src.setBeanFactory(Mockito.mock(BeanFactory.class)); - src.afterPropertiesSet(); - assertEquals(".temp", getPropertyValue(src, "s3Operations.temporaryFileSuffix", String.class)); - } - - - /** - * Instantiates with a custom implementation of {@link AmazonS3Operations} - * which extends from {@link AbstractAmazonS3Operations}. Also sets the following - * s3Operations, directory, fileNameRegex, remoteDirectory, maxObjectsPerBatch and temporarySuffix - * attributes. - */ - @Test - public void withCustomS3Operations() { - BasicAWSCredentials credentials = new BasicAWSCredentials("dummy", "dummy"); - AbstractAmazonS3Operations ops = new AbstractAmazonS3Operations(credentials) { - - @Override - protected void doPut(String bucket, String key, File file, - AmazonS3ObjectACL objectACL, Map userMetadata, - String stringContentMD5) { - - } - - @Override - protected PaginatedObjectsView doListObjects(String bucketName, - String nextMarker, int pageSize, String prefix) { - return null; - } - - @Override - protected AmazonS3Object doGetObject(String bucketName, String key) { - return null; - } - }; - - AmazonS3InboundSynchronizationMessageSource src = new AmazonS3InboundSynchronizationMessageSource(); - src.setS3Operations(ops); - src.setBucket("testbucket"); - src.setDirectory(new LiteralExpression(temp.getRoot().getAbsolutePath())); - src.setFileNameRegex("[a-z]+\\.txt"); - src.setRemoteDirectory("remotedirectory"); - src.setMaxObjectsPerBatch(15); - src.setTemporarySuffix(".temp"); - src.setAcceptSubFolders(true); - src.setDirectory(new LiteralExpression(temp.getRoot().getAbsolutePath())); - src.setBeanFactory(Mockito.mock(BeanFactory.class)); - src.afterPropertiesSet(); - - assertEquals(ops, getPropertyValue(src, "s3Operations", AmazonS3Operations.class)); - assertEquals(".temp", getPropertyValue(src, "s3Operations.temporaryFileSuffix", String.class)); - assertEquals("testbucket", getPropertyValue(src, "bucket", String.class)); - assertEquals(temp.getRoot(), getPropertyValue(src, "directory")); - assertEquals("[a-z]+\\.txt", getPropertyValue(src, "synchronizer.fileNameRegex", String.class)); - assertEquals(true, getPropertyValue(src, "synchronizer.acceptSubFolders", Boolean.class)); - assertEquals(15, getPropertyValue(src, "synchronizer.maxObjectsPerBatch", Integer.class).intValue()); - assertEquals("remotedirectory", getPropertyValue(src, "remoteDirectory", String.class)); - } - - /** - * Synchronizes the local directory to a remote bucket - */ - @Test - public void synchronizeWithLocalDirectory() { - mockAmazonS3Operations(Arrays.asList( - new String[] {"test.txt", "test.txt", md5Hash("test.txt"), null}, - new String[] {"sub1/test.txt", "sub1/test.txt", md5Hash("sub1/test.txt"), null}, - new String[] {"sub1/sub11/test.txt", "sub1/sub11/test.txt", md5Hash("sub1/sub11/test.txt"), null}, - new String[] {"sub2/test.txt", "sub2/test.txt", md5Hash("sub2/test.txt"), null} - )); - AmazonS3InboundSynchronizationMessageSource src = new AmazonS3InboundSynchronizationMessageSource(); - src.setS3Operations(operations); - src.setBucket(AmazonS3OperationsMockingUtil.BUCKET); - src.setDirectory(new LiteralExpression(temp.getRoot().getAbsolutePath())); - src.setFileNameRegex("[a-z]+\\.txt"); - src.setRemoteDirectory("/sub1"); - src.setMaxObjectsPerBatch(15); - src.setTemporarySuffix(".temp"); - src.setAcceptSubFolders(true); - src.setDirectory(new LiteralExpression(temp.getRoot().getAbsolutePath())); - src.setBeanFactory(Mockito.mock(BeanFactory.class)); - src.afterPropertiesSet(); - File file = src.receive().getPayload(); - assertEquals(temp.getRoot().getAbsoluteFile() + - File.separator + "sub1" + File.separator + "test.txt", file.getAbsolutePath()); - file = src.receive().getPayload(); - assertEquals(temp.getRoot().getAbsoluteFile() + - File.separator + "sub1" + File.separator + "sub11" + File.separator + "test.txt", file.getAbsolutePath()); - Message message = src.receive(); - assertNull(message); - } - - - -} diff --git a/src/test/java/org/springframework/integration/aws/s3/AmazonS3ObjectBuilderTests.java b/src/test/java/org/springframework/integration/aws/s3/AmazonS3ObjectBuilderTests.java deleted file mode 100644 index ea79d91..0000000 --- a/src/test/java/org/springframework/integration/aws/s3/AmazonS3ObjectBuilderTests.java +++ /dev/null @@ -1,264 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.aws.s3; - -import static org.springframework.integration.aws.s3.core.ObjectPermissions.READ; -import static org.springframework.integration.aws.s3.core.ObjectPermissions.READ_ACP; -import static org.springframework.integration.aws.s3.core.ObjectPermissions.WRITE_ACP; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; - -import org.junit.Assert; - -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.springframework.integration.aws.s3.core.AmazonS3Object; -import org.springframework.integration.aws.s3.core.AmazonS3ObjectACL; -import org.springframework.integration.aws.s3.core.Grantee; -import org.springframework.integration.aws.s3.core.GranteeType; -import org.springframework.integration.aws.s3.core.ObjectGrant; -import org.springframework.integration.aws.s3.core.ObjectPermissions; -import org.springframework.integration.test.util.TestUtils; - -/** - * The test case for the class {@link AmazonS3ObjectBuilder} - * - * @author Amol Nayak - * - * @since 0.5 - * - */ -public class AmazonS3ObjectBuilderTests { - - @Rule - public TemporaryFolder temp = new TemporaryFolder(); - - private static final String GROUP_GRANTEE = "http://acs.amazonaws.com/groups/global/AllUsers"; - private static final String EMAIL_GRANTEE = "test@test.com"; - private static final String CANONICAL_GRANTEE = "12345678900987654321abcdefabcdeab12345678900987654321abcdefabcde"; - - /** - * Tries to construct the object with a null file instance - */ - @Test(expected=IllegalArgumentException.class) - public void withNullFile() { - AmazonS3ObjectBuilder.getInstance().fromFile(null); - } - - /** - * Tries to construct the file from a non existent file - */ - @Test(expected=IllegalArgumentException.class) - public void withNonExistentFile() { - AmazonS3ObjectBuilder.getInstance().fromFile(new File("somejunkfile")); - } - - /** - * Tries to construct the file which is a directory - */ - @Test(expected=IllegalArgumentException.class) - public void withADirectory() throws IOException { - File dir = temp.newFolder("tempdir"); - AmazonS3ObjectBuilder.getInstance().fromFile(dir); - dir.delete(); - } - - /** - * Tries to construct the {@link AmazonS3Object} from a null location - */ - @Test(expected=IllegalArgumentException.class) - public void withNullPathString() { - AmazonS3ObjectBuilder.getInstance().fromLocation(null); - } - - /** - * Tries to construct the {@link AmazonS3Object} from a valid file - */ - @Test - public void withValidFile() throws Exception { - File file = temp.newFile("temp.txt"); - String pathname = file.getAbsolutePath(); - AmazonS3ObjectBuilder builder = - AmazonS3ObjectBuilder.getInstance().fromLocation(pathname); - file = TestUtils.getPropertyValue(builder, "file", File.class); - Assert.assertNotNull(file); - Assert.assertEquals(pathname, file.getAbsolutePath()); - } - - /** - * Tries to construct the {@link AmazonS3Object} from an {@link InputStream} - */ - @Test - public void withInputStream()throws Exception { - File tempFile = temp.newFile("Temp.txt"); - InputStream in = new FileInputStream(tempFile); - AmazonS3ObjectBuilder builder = - AmazonS3ObjectBuilder.getInstance().fromInputStream(in); - Assert.assertNotNull(TestUtils.getPropertyValue(builder, "in", InputStream.class)); - in.close(); - } - - /** - * Tries to construct the {@link AmazonS3Object} from an {@link InputStream} and a {@link File} - * instance - */ - @Test(expected=IllegalArgumentException.class) - public void withBothinputStreamAndFile() throws Exception { - File tempFile = temp.newFile("Temp.txt"); - FileInputStream in = new FileInputStream(tempFile); - AmazonS3ObjectBuilder - .getInstance() - .fromInputStream(in) - .fromFile(tempFile); - in.close(); - } - - /** - * Constructs the {@link AmazonS3Object} with some user metadata - */ - @Test - public void withUserMetadata() throws Exception { - File tempFile = temp.newFile("Temp.txt"); - AmazonS3ObjectBuilder builder = AmazonS3ObjectBuilder - .getInstance() - .fromFile(tempFile) - .withUserMetaData(Collections.singletonMap("Key", "Value")); - Assert.assertNotNull(TestUtils.getPropertyValue(builder, "userMetaData", Map.class)); - } - - /** - * Constructs the {@link AmazonS3Object} with some metadata - */ - @Test - public void withMetadata() throws Exception { - File tempFile = temp.newFile("Temp.txt"); - AmazonS3ObjectBuilder builder = AmazonS3ObjectBuilder - .getInstance() - .fromFile(tempFile) - .withMetaData(Collections.singletonMap("Key", (Object)"Value")); - Assert.assertNotNull(TestUtils.getPropertyValue(builder, "metaData", Map.class)); - } - - /** - * Constructs the {@link AmazonS3Object} with an invalid ACL identifier - */ - @Test - public void withValidACL() throws Exception { - File tempFile = temp.newFile("Temp.txt"); - Map> acls = generateObjectACLS(); - AmazonS3ObjectBuilder builder = AmazonS3ObjectBuilder - .getInstance() - .fromFile(tempFile) - .withObjectACL(acls); - AmazonS3ObjectACL acl = TestUtils.getPropertyValue(builder, "objectACL", AmazonS3ObjectACL.class); - assertGrants(acl); - } - - /** - * Builds a complete object with all the possible attributes and checks - * if those are properly populated in the constructed {@link AmazonS3Object} - */ - @Test - public void withCompleteObjectFromFile() throws Exception { - File tempFile = temp.newFile("Temp.txt"); - Map> acls = generateObjectACLS(); - Map userMetadata = Collections.singletonMap("Key", "Value"); - Map metadata = Collections.singletonMap("Key", (Object)"Value"); - AmazonS3ObjectBuilder builder = AmazonS3ObjectBuilder - .getInstance() - .fromFile(tempFile) - .withObjectACL(acls) - .withMetaData(metadata) - .withUserMetaData(userMetadata); - AmazonS3Object object = builder.build(); - assertGrants(object.getObjectACL()); - Assert.assertEquals(userMetadata, object.getUserMetaData()); - Assert.assertEquals(metadata, object.getMetaData()); - Assert.assertEquals(tempFile, object.getFileSource()); - Assert.assertNull(object.getInputStream()); - } - - /** - * Builds an object with an {@link InputStream} - */ - @Test - public void wothObjectFromStream() throws Exception { - File tempFile = temp.newFile("Temp.txt"); - InputStream in = new FileInputStream(tempFile); - AmazonS3ObjectBuilder builder = AmazonS3ObjectBuilder - .getInstance() - .fromInputStream(in); - AmazonS3Object object = builder.build(); - Assert.assertNull(object.getFileSource()); - Assert.assertEquals(in,object.getInputStream()); - in.close(); - } - - - - /** - * Generate the object test ACLS - */ - private Map> generateObjectACLS() { - Map> acls = new HashMap>(); - acls.put(CANONICAL_GRANTEE, - Arrays.asList("write acp")); - acls.put(EMAIL_GRANTEE, - Arrays.asList("read acp", "write acp")); - acls.put(GROUP_GRANTEE, - Arrays.asList("read")); - return acls; - } - - /** - * Checks for the grants in the object - * - */ - private void assertGrants(AmazonS3ObjectACL acl) { - Set grants = acl.getGrants(); - Assert.assertEquals(4, grants.size()); - for(ObjectGrant grant:grants) { - Grantee grantee = grant.getGrantee(); - GranteeType type = grantee.getGranteeType(); - ObjectPermissions permission = grant.getPermission(); - if(type == GranteeType.CANONICAL_GRANTEE_TYPE) { - Assert.assertEquals(CANONICAL_GRANTEE, grantee.getIdentifier()); - Assert.assertEquals(WRITE_ACP,permission); - } - else if(type == GranteeType.EMAIL_GRANTEE_TYPE){ - Assert.assertEquals(EMAIL_GRANTEE, grantee.getIdentifier()); - Assert.assertTrue(permission == WRITE_ACP || permission == READ_ACP); - } - else if(type == GranteeType.GROUP_GRANTEE_TYPE) { - Assert.assertEquals(GROUP_GRANTEE, grantee.getIdentifier()); - Assert.assertEquals(READ,permission); - } - } - } - - - -} diff --git a/src/test/java/org/springframework/integration/aws/s3/AmazonS3OperationsMockingUtil.java b/src/test/java/org/springframework/integration/aws/s3/AmazonS3OperationsMockingUtil.java deleted file mode 100644 index 8079c18..0000000 --- a/src/test/java/org/springframework/integration/aws/s3/AmazonS3OperationsMockingUtil.java +++ /dev/null @@ -1,189 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.aws.s3; - -import static org.mockito.Matchers.anyInt; -import static org.mockito.Matchers.anyString; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.when; -import static org.springframework.integration.aws.s3.InboundFileSynchronizationImpl.CONTENT_MD5; - -import java.io.ByteArrayInputStream; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.commons.codec.binary.Base64; -import org.apache.commons.codec.binary.Hex; -import org.mockito.invocation.InvocationOnMock; -import org.mockito.stubbing.Answer; -import org.springframework.integration.aws.s3.core.AmazonS3Object; -import org.springframework.integration.aws.s3.core.AmazonS3Operations; -import org.springframework.integration.aws.s3.core.PaginatedObjectsView; -import org.springframework.integration.aws.s3.core.S3ObjectSummary; -import org.springframework.util.StringUtils; - -/** - * The utility class for mocking the {@link AmazonS3Operations} - * - * @author Amol Nayak - * - * @since 0.5 - */ -public final class AmazonS3OperationsMockingUtil { - - public static final String BUCKET = "com.si.aws.test.bucket"; - private static final List summary = new ArrayList(); - private static final Map objectDetails = new HashMap(); - - private AmazonS3OperationsMockingUtil() { - throw new AssertionError("Cannot instantiate utility class"); - } - - public static final AmazonS3Operations mockS3Operations() { - - AmazonS3Operations operations; - - PaginatedObjectsView view = new PaginatedObjectsView() { - - @Override - public boolean hasMoreResults() { - return false; - } - - @Override - public List getObjectSummary() { - return summary; - } - - @Override - public String getNextMarker() { - return null; - } - }; - operations = mock(AmazonS3Operations.class); - when(operations.listObjects(anyString(), anyString(), anyString(), anyInt())) - .thenReturn(view); - - when(operations.getObject(anyString(), anyString(), anyString())) - .then(new Answer() { - public AmazonS3Object answer(InvocationOnMock invocation) - throws Throwable { - String folderName = (String)invocation.getArguments()[1]; - String fileName = (String)invocation.getArguments()[2]; - - if(StringUtils.hasText(folderName)) { - if(folderName.startsWith("/")) { - folderName = folderName.substring(1); - } - if(folderName.endsWith("/")) { - folderName = folderName + "/"; - } - } - else { - folderName = ""; - } - - - String[] object = objectDetails.get(folderName + fileName); - AmazonS3Object s3Object; - if(object != null) { - s3Object = new AmazonS3ObjectBuilder() - .fromInputStream(new ByteArrayInputStream(object[1].getBytes())) - .withUserMetaData(Collections.singletonMap(CONTENT_MD5, object[2])) - .build(); - } - else { - s3Object = null; - } - - return s3Object; - } - }); - - return operations; - } - - - /** - * The private helper method that is used to mock the {@link AmazonS3Operations#listObjects(String, String, String, int) - * method - * - * The method accepts a List of object[] where each element of object array has the - * following significance - * - * object[0] is the key of the file - * object[1] is the content of the file - * object[2] is the MD5 content to be used (optional) - * object[3] is the etag of the object - * - * @param listObjects - */ - public static void mockAmazonS3Operations(final List listObjects) { - summary.clear(); - for(String[] listObject:listObjects) { - addToSummaryList(listObject); - objectDetails.put(listObject[0], listObject); - } - } - - /** - * Creates a {@link S3ObjectSummary} with the given details and adds it to the summary object list - * @param listObject - */ - private static void addToSummaryList(final String[] listObject) { - summary.add( - new S3ObjectSummary() { - - @Override - public long getSize() { - return 0; - } - - @Override - public Date getLastModified() { - return null; - } - - @Override - public String getKey() { - return listObject[0]; - } - - @Override - public String getETag() { - if(listObject[3] != null) { - return listObject[3]; - } - else { - byte[] b64 = Base64.decodeBase64((listObject[2]).getBytes()); - return Hex.encodeHexString(b64); - } - - } - - @Override - public String getBucketName() { - return null; - } - } - ); - } - -} diff --git a/src/test/java/org/springframework/integration/aws/s3/DefaultFileNameGenerationStrategyTests.java b/src/test/java/org/springframework/integration/aws/s3/DefaultFileNameGenerationStrategyTests.java deleted file mode 100644 index 60fbb1e..0000000 --- a/src/test/java/org/springframework/integration/aws/s3/DefaultFileNameGenerationStrategyTests.java +++ /dev/null @@ -1,126 +0,0 @@ -/* - * Copyright 2002-2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.aws.s3; - -import java.io.File; -import java.io.IOException; -import java.util.UUID; - -import org.junit.Assert; -import org.junit.Rule; - -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.mockito.Mockito; - -import org.springframework.beans.factory.BeanFactory; -import org.springframework.integration.support.MessageBuilder; -import org.springframework.messaging.Message; - - -/** - * The test class for {@link DefaultFileNameGenerationStrategy} - * - * @author Amol Nayak - * @author Rob Harrop - * @author Artem Bilan - * - * @since 0.5 - * - */ -public class DefaultFileNameGenerationStrategyTests { - - @Rule - public TemporaryFolder tempFolder = new TemporaryFolder(); - - /** - * Tests with the file name present in the predetermined header "file_name" of the message - */ - @Test - public void withNameInHeader() { - Message message = MessageBuilder.withPayload("SomeString") - .setHeader(AmazonS3MessageHeaders.FILE_NAME, "FileName.txt") - .build(); - DefaultFileNameGenerationStrategy strategy = new DefaultFileNameGenerationStrategy(); - strategy.setBeanFactory(Mockito.mock(BeanFactory.class)); - Assert.assertEquals("FileName.txt", strategy.generateFileName(message)); - } - - /** - * Tests with a payload as a temp file payload - * @throws IOException - */ - @Test - public void withATempFile() throws IOException { - final File file = tempFolder.newFile("TempFile.txt.writing"); - Message message = MessageBuilder.withPayload(file) - .build(); - DefaultFileNameGenerationStrategy strategy = new DefaultFileNameGenerationStrategy(); - strategy.setBeanFactory(Mockito.mock(BeanFactory.class)); - Assert.assertEquals("TempFile.txt", strategy.generateFileName(message)); - file.delete(); - } - - /** - * Tests with a payload as a temp file payload - * @throws IOException - */ - @Test - public void withANonTempFile() throws IOException { - final File file = tempFolder.newFile("TempFile.txt"); - Message message = MessageBuilder.withPayload(file) - .build(); - DefaultFileNameGenerationStrategy strategy = new DefaultFileNameGenerationStrategy(); - strategy.setBeanFactory(Mockito.mock(BeanFactory.class)); - Assert.assertEquals("TempFile.txt", strategy.generateFileName(message)); - file.delete(); - } - - /** - * Tests with a payload as a temp file payload - */ - @Test - public void withMessageIdName() { - Message message = MessageBuilder.withPayload("String") - .build(); - DefaultFileNameGenerationStrategy strategy = new DefaultFileNameGenerationStrategy(); - strategy.setBeanFactory(Mockito.mock(BeanFactory.class)); - UUID uid = message.getHeaders().getId(); - Assert.assertEquals(uid.toString() + ".ext", strategy.generateFileName(message)); - } - - /** - * Tests with the file name generation expression as a null value - */ - @Test(expected=IllegalArgumentException.class) - public void withNullExprssion() { - DefaultFileNameGenerationStrategy strategy = new DefaultFileNameGenerationStrategy(); - strategy.setBeanFactory(Mockito.mock(BeanFactory.class)); - strategy.setFileNameExpression(null); - } - - /** - * Tests with a null value for temporary suffix - */ - @Test(expected=IllegalArgumentException.class) - public void withNullTemporarySuffix() { - DefaultFileNameGenerationStrategy strategy = new DefaultFileNameGenerationStrategy(); - strategy.setBeanFactory(Mockito.mock(BeanFactory.class)); - strategy.setTemporarySuffix(null); - } - -} diff --git a/src/test/java/org/springframework/integration/aws/s3/FileNameFilterTests.java b/src/test/java/org/springframework/integration/aws/s3/FileNameFilterTests.java deleted file mode 100644 index d112b34..0000000 --- a/src/test/java/org/springframework/integration/aws/s3/FileNameFilterTests.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.aws.s3; - -import org.junit.Assert; - -import org.junit.Test; - -/** - * - * The test cases for various {@link FileNameFilter} implementations. - * - * @author Amol Nayak - * - * @since 0.5 - * - */ -public class FileNameFilterTests { - - /** - * The case sets the folder of the filter to null so that it accepts all the files in the same - * folder, the {@link AbstractFileNameFilter#isAcceptSubFolders() returns false. - * - */ - @Test - public void acceptAllFilesWithoutSubfolders() { - AbstractFileNameFilter filter = new AlwaysTrueFileNamefilter(); - filter.setAcceptSubFolders(false); - Assert.assertTrue(filter.accept("SomeFile.txt")); - Assert.assertFalse(filter.accept("somessubfolder/SomeFile.txt")); - } - - - /** - * The case sets the folder of the filter to null so that it accepts all the files in the same - * folder and the sub folders, the {@link AbstractFileNameFilter#isAcceptSubFolders() returns true. - * - */ - @Test - public void acceptAllFilesWithSubfolders() { - AbstractFileNameFilter filter = new AlwaysTrueFileNamefilter(); - filter.setAcceptSubFolders(true); - Assert.assertTrue(filter.accept("SomeFile.txt")); - Assert.assertTrue(filter.accept("somessubfolder/SomeFile.txt")); - } - - /** - * The test case sets a sub folder for the search and sets the - * {@link AbstractFileNameFilter#setAcceptSubFolders(boolean) to false - */ - @Test - public void acceptInSubfolderWithoutSubfolder() { - AbstractFileNameFilter filter = new AlwaysTrueFileNamefilter(); - filter.setFolderName("/subfolder"); - filter.setAcceptSubFolders(false); - Assert.assertFalse(filter.accept("FileName.txt")); - Assert.assertTrue(filter.accept("subfolder/FileName.txt")); - Assert.assertFalse(filter.accept("subfolder/anothersf/FileName.txt")); - } - - /** - * The test case sets a sub folder for the search and sets the - * {@link AbstractFileNameFilter#setAcceptSubFolders(boolean) to true - */ - @Test - public void acceptInSubfolderWithSubfolder() { - AbstractFileNameFilter filter = new AlwaysTrueFileNamefilter(); - filter.setFolderName("/subfolder"); - filter.setAcceptSubFolders(true); - Assert.assertFalse(filter.accept("FileName.txt")); - Assert.assertTrue(filter.accept("subfolder/FileName.txt")); - Assert.assertTrue(filter.accept("subfolder/anothersf/FileName.txt")); - } - - /** - * Tests the regex file filter - */ - @Test - public void regexTest() { - //accept only file names with name in lower case and ends with .txt - AbstractFileNameFilter filter = new RegexFileNameFilter("[a-z]+\\.txt"); - filter.setAcceptSubFolders(true); - Assert.assertTrue(filter.accept("test.txt")); - Assert.assertFalse(filter.accept("Test.txt")); - Assert.assertFalse(filter.accept("test123.txt")); - Assert.assertFalse(filter.accept("test.tx")); - Assert.assertFalse(filter.accept("test")); - Assert.assertTrue(filter.accept("test/test.txt")); - Assert.assertTrue(filter.accept("test/Test/12/test.txt")); - Assert.assertFalse(filter.accept("test/Test/12/test.tx")); - Assert.assertFalse(filter.accept("test/Test/12/Test.txt")); - } - - /** - * Tests the wildcard file filter - */ - @Test - public void wildCardTest() { - //accept only file names with name in lower case and ends with .txt - AbstractFileNameFilter filter = new WildcardFileNameFilter("*.txt"); - filter.setAcceptSubFolders(true); - Assert.assertTrue(filter.accept("test.txt")); - Assert.assertTrue(filter.accept("Test.txt")); - Assert.assertTrue(filter.accept("test123.txt")); - Assert.assertFalse(filter.accept("test.tx")); - Assert.assertFalse(filter.accept("test")); - Assert.assertTrue(filter.accept("test/test.txt")); - Assert.assertTrue(filter.accept("test/Test/12/test.txt")); - Assert.assertTrue(filter.accept("test/Test/12/Test.txt")); - Assert.assertFalse(filter.accept("test/Test/12/Test.ext")); - } -} diff --git a/src/test/java/org/springframework/integration/aws/s3/InboundFileSynchronizationImplTests.java b/src/test/java/org/springframework/integration/aws/s3/InboundFileSynchronizationImplTests.java deleted file mode 100644 index 6cd9550..0000000 --- a/src/test/java/org/springframework/integration/aws/s3/InboundFileSynchronizationImplTests.java +++ /dev/null @@ -1,462 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.aws.s3; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.springframework.integration.aws.common.AWSTestUtils.assertFileContent; -import static org.springframework.integration.aws.common.AWSTestUtils.getContentsRecursively; -import static org.springframework.integration.aws.common.AWSTestUtils.md5Hash; -import static org.springframework.integration.aws.common.AWSTestUtils.writeToFile; -import static org.springframework.integration.aws.s3.AmazonS3OperationsMockingUtil.BUCKET; -import static org.springframework.integration.aws.s3.AmazonS3OperationsMockingUtil.mockAmazonS3Operations; -import static org.springframework.integration.aws.s3.AmazonS3OperationsMockingUtil.mockS3Operations; - -import java.io.File; -import java.util.Arrays; - -import org.apache.commons.codec.binary.Base64; -import org.apache.commons.codec.binary.Hex; -import org.junit.BeforeClass; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.springframework.integration.aws.s3.core.AmazonS3Operations; -import org.springframework.integration.test.util.TestUtils; - -/** - * Test class for {@link InboundFileSynchronizationImpl} - * - * @author Amol Nayak - * - * @since 0.5 - * - */ -public class InboundFileSynchronizationImplTests { - - private static AmazonS3Operations operations; - - @Rule - public final TemporaryFolder tempFolder = new TemporaryFolder(); - - @BeforeClass - public static void setup() { - operations = mockS3Operations(); - } - - private InboundFileSynchronizationImpl getInboundFileSynchronizationImpl() { - //uncomment the below code if you want to execute against the actual bucket - //but before that you need to do the following - //create a bucket and set that in the constant BUCKET in the class - //1. add a file test.txt with content test.txt to the root. - //2. create a folder sub1 and add a file test.txt to it with content sub1/test.txt. - //3. create a folder sub1/sub11 and add a file test.txt to it with content sub1/sub11/test.txt. - //4. create a folder sub2 and add a file test.txt to it with content sub/test.txt. - -// DefaultAmazonS3Operations operations = new DefaultAmazonS3Operations(AmazonWSTestUtils.getCredentials()); -// try { -// operations.afterPropertiesSet(); -// } catch (Exception e) { -// e.printStackTrace(); -// } - - InboundFileSynchronizationImpl sync = new InboundFileSynchronizationImpl(operations, - new InboundLocalFileOperationsImpl()); - return sync; - } - - - /** - * Tests with {@link AmazonS3Operations} instance as null - */ - @Test(expected=IllegalArgumentException.class) - public void withS3OperationsAsNull() { - new InboundFileSynchronizationImpl(null, new InboundLocalFileOperationsImpl()); - } - - /** - * Tests with {@link InboundLocalFileOperations} instance as null - */ - @Test(expected=IllegalArgumentException.class) - public void withLocalFileOperationsAsNull() { - new InboundFileSynchronizationImpl(operations, null); - } - - /** - * Tests after setting both the wildcard and filename regex - */ - @Test(expected=IllegalArgumentException.class) - public void withBothWildCardAndRegex() throws Exception { - InboundFileSynchronizationImpl sync = getInboundFileSynchronizationImpl(); - sync.setFileWildcard("*.txt"); - sync.setFileNamePattern("[a-z]+\\.txt"); - sync.afterPropertiesSet(); - } - - /** - * Sets non of regex and wildcard - */ - @Test - public void withNoneOfRegexAndWildcard() throws Exception { - InboundFileSynchronizationImpl impl = getInboundFileSynchronizationImpl(); - impl.afterPropertiesSet(); - assertEquals(AlwaysTrueFileNamefilter.class, - TestUtils.getPropertyValue(impl, "filter", FileNameFilter.class).getClass()); - } - - /** - * Tests after setting filename regex only - */ - @Test - public void withRegexOnly() throws Exception { - InboundFileSynchronizationImpl sync = getInboundFileSynchronizationImpl(); - sync.setFileNamePattern("[a-z]+\\.txt"); - sync.afterPropertiesSet(); - FileNameFilter filter = TestUtils.getPropertyValue(sync, "filter",FileNameFilter.class); - assertNotNull(filter); - assertEquals(RegexFileNameFilter.class, filter.getClass()); - assertEquals("[a-z]+\\.txt", - TestUtils.getPropertyValue(filter, "filter.pattern.pattern", String.class)); - - } - - /** - * Tests after setting filename wildcard only - */ - @Test - public void withWildcardOnly() throws Exception { - InboundFileSynchronizationImpl sync = getInboundFileSynchronizationImpl(); - sync.setFileWildcard("*.txt"); - sync.afterPropertiesSet(); - FileNameFilter filter = TestUtils.getPropertyValue(sync, "filter",FileNameFilter.class); - assertNotNull(filter); - assertEquals(WildcardFileNameFilter.class, filter.getClass()); - assertEquals("*.txt", - TestUtils.getPropertyValue(filter, "filter.wildcards", String[].class)[0]); - } - - - /** - * Sets the {@link InboundFileSynchronizationImpl#setAcceptSubFolders(boolean) as true - */ - @Test - public void withAcceptSubfolderAsTrue() throws Exception { - InboundFileSynchronizationImpl impl = getInboundFileSynchronizationImpl(); - impl.setAcceptSubFolders(true); - impl.afterPropertiesSet(); - assertTrue(TestUtils.getPropertyValue(impl, "filter.acceptSubFolders",Boolean.class).booleanValue()); - assertTrue(TestUtils.getPropertyValue(impl, "fileOperations.createDirectoriesIfRequired", - Boolean.class).booleanValue()); - } - - - /** - * Invokes with remote directory as / and create directory set to true - */ - @Test - public void withRemoteAsRootAndCreateDirectoryToTrue() throws Exception { - setupMock(); - String rootDirectoryPath = tempFolder.getRoot().getAbsolutePath(); - String path = String.format("%s%s%s", - rootDirectoryPath,File.separator,"test.txt"); - File fileOne = new File(path); - path = String.format("%s%s%s%s%s", - rootDirectoryPath,File.separator,"sub1",File.separator,"test.txt"); - File fileTwo = new File(path); - path = String.format("%s%s%s%s%s%s%s", - rootDirectoryPath,File.separator,"sub1",File.separator,"sub11",File.separator,"test.txt"); - File fileThree = new File(path);; - path = String.format("%s%s%s%s%s", - rootDirectoryPath,File.separator,"sub2",File.separator,"test.txt"); - File fileFour = new File(path); - assertFalse(fileOne.exists()); - assertFalse(fileTwo.exists()); - assertFalse(fileThree.exists()); - assertFalse(fileFour.exists()); - InboundFileSynchronizationImpl impl = getInboundFileSynchronizationImpl(); - impl.setAcceptSubFolders(true); - impl.afterPropertiesSet(); - impl.synchronizeToLocalDirectory(tempFolder.getRoot(), BUCKET, "/"); - assertTrue(fileOne.exists()); - assertFileContent(fileOne, "test.txt"); - assertTrue(fileTwo.exists()); - assertFileContent(fileTwo, "sub1/test.txt"); - assertTrue(fileThree.exists()); - assertFileContent(fileThree, "sub1/sub11/test.txt"); - assertTrue(fileFour.exists()); - assertFileContent(fileFour, "sub2/test.txt"); - assertEquals(4, getContentsRecursively(tempFolder.getRoot()).size()); - } - - - /** - * Invokes with remote directory as / and create directory set to false - */ - @Test - public void withRemoteAsRootAndCreateDirectoryToFalse() throws Exception { - setupMock(); - String rootDirectoryPath = tempFolder.getRoot().getAbsolutePath(); - String path = String.format("%s%s%s", - rootDirectoryPath,File.separator,"test.txt"); - File fileOne = new File(path); - assertFalse(fileOne.exists()); - InboundFileSynchronizationImpl impl = getInboundFileSynchronizationImpl(); - impl.setAcceptSubFolders(false); - impl.afterPropertiesSet(); - impl.synchronizeToLocalDirectory(tempFolder.getRoot(), BUCKET, "/"); - assertTrue(fileOne.exists()); - assertFileContent(fileOne, "test.txt"); - assertEquals(1, getContentsRecursively(tempFolder.getRoot()).size()); - } - - - /** - * Invokes with remote directory as /sub1 and create directory set to true - */ - @Test - public void withRemoteAssub1AndCreateDirectoryToTrue() throws Exception { - setupMock(); - String rootDirectoryPath = tempFolder.getRoot().getAbsolutePath(); - String path = String.format("%s%s%s%s%s", - rootDirectoryPath,File.separator,"sub1",File.separator,"test.txt"); - File fileOne = new File(path); - path = String.format("%s%s%s%s%s%s%s", - rootDirectoryPath,File.separator,"sub1",File.separator,"sub11",File.separator,"test.txt"); - File fileTwo = new File(path); - assertFalse(fileOne.exists()); - assertFalse(fileTwo.exists()); - InboundFileSynchronizationImpl impl = getInboundFileSynchronizationImpl(); - impl.setAcceptSubFolders(true); - impl.afterPropertiesSet(); - impl.synchronizeToLocalDirectory(tempFolder.getRoot(), BUCKET, "/sub1"); - assertTrue(fileOne.exists()); - assertFileContent(fileOne, "sub1/test.txt"); - assertTrue(fileTwo.exists()); - assertFileContent(fileTwo, "sub1/sub11/test.txt"); - assertEquals(2, getContentsRecursively(tempFolder.getRoot()).size()); - } - - /** - * Invokes with remote directory as /sub1 and create directory set to false - */ - @Test - public void withRemoteAssub1AndCreateDirectoryToFalse() throws Exception { - setupMock(); - String rootDirectoryPath = tempFolder.getRoot().getAbsolutePath(); - String path = String.format("%s%s%s%s%s", - rootDirectoryPath,File.separator,"sub1",File.separator,"test.txt"); - File file = new File(path); - assertFalse(file.exists()); - InboundFileSynchronizationImpl impl = getInboundFileSynchronizationImpl(); - impl.setAcceptSubFolders(false); - impl.afterPropertiesSet(); - impl.synchronizeToLocalDirectory(tempFolder.getRoot(), BUCKET, "/sub1"); - assertTrue(file.exists()); - assertFileContent(file, "sub1/test.txt"); - assertEquals(1, getContentsRecursively(tempFolder.getRoot()).size()); - } - - /** - * Invokes with remote directory as / and create directory set to false - * The two files test.txt and sub1/test.txt would already be present on - * the file system. Both test.txt and sub/test.txt will have content different that the remote one. - * test.txt will be replaced and sub/test.txt will not be replaced. - * - */ - @Test - public void withRemoteAsRootAndCreateDirectoryToFalse2() throws Exception { - setupMock(); - String rootDirectoryPath = tempFolder.getRoot().getAbsolutePath(); - //create test.txt and sub1/test.txt - - String path = String.format("%s%s%s", - rootDirectoryPath,File.separator,"test.txt"); - writeToFile(path, "OldContents"); - File fileOne = new File(path); - assertTrue(fileOne.exists()); - assertFileContent(fileOne, "OldContents"); - - path = String.format("%s%s%s%s%s", - rootDirectoryPath,File.separator,"sub1",File.separator,"test.txt"); - writeToFile(path, "OldContents"); - File fileTwo = new File(path); - assertTrue(fileTwo.exists()); - assertFileContent(fileTwo, "OldContents"); - - InboundFileSynchronizationImpl impl = getInboundFileSynchronizationImpl(); - impl.setAcceptSubFolders(false); - impl.afterPropertiesSet(); - impl.synchronizeToLocalDirectory(tempFolder.getRoot(), BUCKET, "/"); - assertTrue(fileOne.exists()); - assertFileContent(fileOne, "test.txt"); - - assertTrue(fileTwo.exists()); - assertFileContent(fileTwo, "OldContents"); - - assertEquals(2, getContentsRecursively(tempFolder.getRoot()).size()); - } - - /** - * Invokes with remote directory as / and create directory set to true - * The two files test.txt and sub1/test.txt would already be present on - * the file system. Both test.txt and sub/test.txt will have content different that the remote one. - * test.txt and sub/test.txt both will be replaced. - * - */ - @Test - public void withRemoteAsRootAndCreateDirectoryToTrue2() throws Exception { - setupMock(); - String rootDirectoryPath = tempFolder.getRoot().getAbsolutePath(); - //create test.txt and sub1/test.txt - - String path = String.format("%s%s%s", - rootDirectoryPath,File.separator,"test.txt"); - writeToFile(path, "OldContents"); - File fileOne = new File(path); - assertTrue(fileOne.exists()); - assertFileContent(fileOne, "OldContents"); - - path = String.format("%s%s%s%s%s", - rootDirectoryPath,File.separator,"sub1",File.separator,"test.txt"); - writeToFile(path, "OldContents"); - File fileTwo = new File(path); - assertTrue(fileTwo.exists()); - assertFileContent(fileTwo, "OldContents"); - - InboundFileSynchronizationImpl impl = getInboundFileSynchronizationImpl(); - impl.setAcceptSubFolders(true); - impl.afterPropertiesSet(); - impl.synchronizeToLocalDirectory(tempFolder.getRoot(), BUCKET, "/"); - assertTrue(fileOne.exists()); - assertFileContent(fileOne, "test.txt"); - - assertTrue(fileTwo.exists()); - assertFileContent(fileTwo, "sub1/test.txt"); - - assertEquals(4, getContentsRecursively(tempFolder.getRoot()).size()); - } - - /** - * The case is slightly different then previous ones. - * We list from the /sub1 with /sub1/test.txt and /sub1/sub11/test.txt - * having different contents, however the etag of - * /sub1/sub11/test.txt is same as remote one hence the local one should not get replaced - * where as /sub1/test.txt should. - * - */ - @Test - public void withRemoteAsRootAndCreateDirectoryToTrue3() throws Exception { - mockAmazonS3Operations(Arrays.asList( - new String[]{"test.txt","test.txt",md5Hash("test.txt"),null}, - new String[]{"sub1/test.txt","sub1/test.txt",md5Hash("sub1/test.txt"),null}, - new String[]{"sub1/sub11/test.txt","sub1/sub11/test.txt",md5Hash("OldContents"),null}, - new String[]{"sub2/test.txt","sub2/test.txt",md5Hash("sub2/test.txt"),null} - )); - - withinSub1FolderTests(true); - - } - - /** - * The scenario tests by listing the directory /sub1 which has two files - * /sub1/test.txt and /sub1/sub11/test.txt. The MD5 of the file will be absent and the etag is - * for MultiUpload. This should force replace the file irrespective of the content. - */ - @Test - public void withMultipartUploadForceReplace() throws Exception { - mockAmazonS3Operations(Arrays.asList( - new String[]{"sub1/test.txt","sub1/test.txt",null, - new String(Hex.encodeHex(Base64.decodeBase64(md5Hash("SomeContentSub1").getBytes()))) + "-1"}, - new String[]{"sub1/sub11/test.txt","sub1/sub11/test.txt",null, - new String(Hex.encodeHex(Base64.decodeBase64(md5Hash("SomeContentSub1/Sub11").getBytes()))) + "-1"} - )); - withinSub1FolderTests(false); - - } - - /** - * The scenario will test with two files present in /sub1 directory, /sub1/test.txt and - * /sub1/sub11/test.txt. Now both these files have multipart upload etag but both have - * MD5 hash in the user's metadata. The contents of both the files is different than the one - * on remote but /sub/test.txt has MD5 sum same as remote, so this should not get replaced - * - * @throws Exception - */ - @Test - public void withMultipartUploadWithMD5Metadata() throws Exception { - mockAmazonS3Operations(Arrays.asList( - new String[]{"sub1/test.txt","sub1/test.txt",md5Hash("sub1/test.txt"), - new String(Hex.encodeHex(Base64.decodeBase64(md5Hash("sub1/test.txt").getBytes()))) + "-1"}, - new String[]{"sub1/sub11/test.txt","sub1/sub11/test.txt",md5Hash("OldContents"), - new String(Hex.encodeHex(Base64.decodeBase64(md5Hash("OldContents").getBytes()))) + "-1"} - )); - - withinSub1FolderTests(true); - } - - /** - * Private method that extracts the common assertion logic for the files in sub1 folder - * @throws Exception - */ - private void withinSub1FolderTests(boolean acceptSubfolder) throws Exception { - String rootDirectoryPath = tempFolder.getRoot().getAbsolutePath(); - - //create sub1/sub11/test.txt and sub1/test.txt - String path = String.format("%s%s%s%s%s%s%s", - rootDirectoryPath,File.separator,"sub1",File.separator,"sub11",File.separator,"test.txt"); - writeToFile(path, "OldContents"); - File fileOne = new File(path); - assertTrue(fileOne.exists()); - assertFileContent(fileOne, "OldContents"); - - path = String.format("%s%s%s%s%s", - rootDirectoryPath,File.separator,"sub1",File.separator,"test.txt"); - writeToFile(path, "OldContents"); - File fileTwo = new File(path); - assertTrue(fileTwo.exists()); - assertFileContent(fileTwo, "OldContents"); - - InboundFileSynchronizationImpl impl = getInboundFileSynchronizationImpl(); - impl.setAcceptSubFolders(acceptSubfolder); - impl.afterPropertiesSet(); - impl.synchronizeToLocalDirectory(tempFolder.getRoot(), BUCKET, "/sub1"); - - assertTrue(fileOne.exists()); - assertFileContent(fileOne, "OldContents"); - - assertTrue(fileTwo.exists()); - assertFileContent(fileTwo, "sub1/test.txt"); - - assertEquals(2, getContentsRecursively(tempFolder.getRoot()).size()); - } - - - /** - * Private helper method that will be setup mock s3 operations to give an illusion - * that it has 4 objects in the remote bucket - * - */ - private void setupMock() { - mockAmazonS3Operations(Arrays.asList( - new String[]{"test.txt","test.txt",md5Hash("test.txt"),null}, - new String[]{"sub1/test.txt","sub1/test.txt",md5Hash("sub1/test.txt"),null}, - new String[]{"sub1/sub11/test.txt","sub1/sub11/test.txt",md5Hash("sub1/sub11/test.txt"),null}, - new String[]{"sub2/test.txt","sub2/test.txt",md5Hash("sub2/test.txt"),null} - )); - } -} diff --git a/src/test/java/org/springframework/integration/aws/s3/InboundLocalFileOperationsImplTests.java b/src/test/java/org/springframework/integration/aws/s3/InboundLocalFileOperationsImplTests.java deleted file mode 100644 index 9ead4d8..0000000 --- a/src/test/java/org/springframework/integration/aws/s3/InboundLocalFileOperationsImplTests.java +++ /dev/null @@ -1,177 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.aws.s3; - -import static org.springframework.integration.aws.common.AWSTestUtils.assertFileContent; - -import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.FileOutputStream; -import java.util.Collections; - -import org.junit.Assert; - -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.springframework.integration.test.util.TestUtils; - -/** - * The test class for {@link InboundLocalFileOperationsImpl} that is used to perform - * operations on local file system - * - * @author Amol Nayak - * - * @since 0.5 - * - */ -public class InboundLocalFileOperationsImplTests { - - @Rule - public TemporaryFolder tempFolder = new TemporaryFolder(); - - /** - * Tries registering a null listener with the class - */ - @Test(expected=IllegalArgumentException.class) - public void withNullListener() { - InboundLocalFileOperations operations = new InboundLocalFileOperationsImpl(); - operations.addEventListener(null); - } - - /** - * Tries setting the listeners which is an empty list - */ - @SuppressWarnings("unchecked") - @Test(expected=IllegalArgumentException.class) - public void setEmptyListeners() { - InboundLocalFileOperations operations = new InboundLocalFileOperationsImpl(); - operations.setEventListeners(Collections.EMPTY_LIST); - } - - /** - * Test case for setting a temporary suffix that begins with a . - */ - @Test - public void setTempSuffixBeginningWithDot() { - InboundLocalFileOperations operations = new InboundLocalFileOperationsImpl(); - operations.setTemporaryFileSuffix(".write"); - Assert.assertEquals(".write", TestUtils.getPropertyValue(operations, "tempFileSuffix")); - } - - /** - * Test case for setting a temporary suffix that does not begins with a . - */ - @Test - public void setTempSuffixNotBeginningWithDot() { - InboundLocalFileOperations operations = new InboundLocalFileOperationsImpl(); - operations.setTemporaryFileSuffix("write"); - Assert.assertEquals(".write", TestUtils.getPropertyValue(operations, "tempFileSuffix")); - } - - - /** - *Since the provided directory is null, we expect an {@link IllegalArgumentException} - */ - @Test(expected=IllegalArgumentException.class) - public void writeWithNullDirectory() throws Exception { - InboundLocalFileOperations operations = new InboundLocalFileOperationsImpl(); - operations.writeToFile(null, null, null); - } - - /** - *Since the provided file name is null, we expect an {@link IllegalArgumentException} - */ - @Test(expected=IllegalArgumentException.class) - public void writeWithNullFileName() throws Exception { - InboundLocalFileOperations operations = new InboundLocalFileOperationsImpl(); - operations.writeToFile(tempFolder.newFolder("Test"), null, null); - } - - /** - *Since the provided stream as null, we expect an {@link IllegalArgumentException} - */ - @Test(expected=IllegalArgumentException.class) - public void writeWithNullStream() throws Exception { - InboundLocalFileOperations operations = new InboundLocalFileOperationsImpl(); - operations.writeToFile(tempFolder.newFolder("Test"), "TestFile.txt", null); - } - - /** - *Provided {@link File} for directory exists and is not a directory - */ - @Test(expected=IllegalArgumentException.class) - public void writeWithExistantNonDirectory() throws Exception { - InboundLocalFileOperations operations = new InboundLocalFileOperationsImpl(); - operations.writeToFile(tempFolder.newFile("Test"), "TestFile.txt", new ByteArrayInputStream(new byte[]{})); - } - - /** - *Provided {@link File} for directory does not exist exists and the create flag is false - */ - @Test(expected=IllegalArgumentException.class) - public void writeWithNonExistentDirectory() throws Exception { - InboundLocalFileOperations operations = new InboundLocalFileOperationsImpl(); - operations.writeToFile(new File(tempFolder.getRoot() + "SomeDir"), "TestFile.txt", new ByteArrayInputStream(new byte[]{})); - } - - /** - * Writes some test content to the file - */ - @Test - public void writeTestContentToFile() throws Exception { - InboundLocalFileOperations operations = new InboundLocalFileOperationsImpl(); - operations.setCreateDirectoriesIfRequired(true); - File directory = new File(tempFolder.getRoot() + File.separator + "someNestedDir"); - File tempFile = new File(directory.getAbsolutePath() + File.separator + "SomeFileName.txt.writing"); - File permFile = new File(directory.getAbsolutePath() + File.separator + "SomeFileName.txt"); - Assert.assertFalse(tempFile.exists()); - Assert.assertFalse(permFile.exists()); - operations.writeToFile(directory, "SomeFileName.txt", new ByteArrayInputStream("Some Test Content".getBytes())); - Assert.assertFalse(tempFile.exists()); - Assert.assertTrue(permFile.exists()); - //Check the content - assertFileContent(permFile, "Some Test Content"); - //TODO: Test FileEventHandlers - } - - /** - * Writes some test content to the file with teh given target file existent - */ - @Test - public void writeTestContentWithTargetExistent() throws Exception { - InboundLocalFileOperations operations = new InboundLocalFileOperationsImpl(); - operations.setCreateDirectoriesIfRequired(true); - File directory = tempFolder.newFolder("someNestedDir"); - File tempFile = new File(directory.getAbsolutePath() + File.separator + "SomeFileName.txt.writing"); - File permFile = new File(directory.getAbsolutePath() + File.separator + "SomeFileName.txt"); - permFile.createNewFile(); - //Write Some content - FileOutputStream fos = new FileOutputStream(permFile); - fos.write("Some Old Contents".getBytes()); - fos.close(); - assertFileContent(permFile, "Some Old Contents"); - Assert.assertFalse(tempFile.exists()); - Assert.assertTrue(permFile.exists()); - operations.writeToFile(directory, "SomeFileName.txt", new ByteArrayInputStream("Some Test Content".getBytes())); - Assert.assertFalse(tempFile.exists()); - Assert.assertTrue(permFile.exists()); - //Check the content - assertFileContent(permFile, "Some Test Content"); - } - - -} diff --git a/src/test/java/org/springframework/integration/aws/s3/config/xml/AmazonS3DummyOperations.java b/src/test/java/org/springframework/integration/aws/s3/config/xml/AmazonS3DummyOperations.java deleted file mode 100644 index e635d55..0000000 --- a/src/test/java/org/springframework/integration/aws/s3/config/xml/AmazonS3DummyOperations.java +++ /dev/null @@ -1,55 +0,0 @@ -/* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.aws.s3.config.xml; - -import org.springframework.integration.aws.s3.core.AmazonS3Object; -import org.springframework.integration.aws.s3.core.AmazonS3Operations; -import org.springframework.integration.aws.s3.core.PaginatedObjectsView; - -/** - * The dummy {@link AmazonS3Operations} for tests - * - * @author Amol Nayak - * - * @since 0.5 - * - */ -public class AmazonS3DummyOperations implements AmazonS3Operations { - - @Override - public PaginatedObjectsView listObjects(String bucketName, - String folder, String nextMarker, int pageSize) { - return null; - } - - @Override - public void putObject(String bucketName, String folder, - String objectName, AmazonS3Object s3Object) { - } - - @Override - public AmazonS3Object getObject(String bucketName, String folder, - String objectName) { - return null; - } - - @Override - public boolean removeObject(String bucketName, String folder, - String objectName) { - return false; - } - -} diff --git a/src/test/java/org/springframework/integration/aws/s3/config/xml/AmazonS3InboundChannelAdapterParserTests.java b/src/test/java/org/springframework/integration/aws/s3/config/xml/AmazonS3InboundChannelAdapterParserTests.java deleted file mode 100644 index 4e7d325..0000000 --- a/src/test/java/org/springframework/integration/aws/s3/config/xml/AmazonS3InboundChannelAdapterParserTests.java +++ /dev/null @@ -1,86 +0,0 @@ -/* - * Copyright 2002-2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.springframework.integration.aws.s3.config.xml; - -import static org.junit.Assert.assertEquals; -import static org.springframework.integration.test.util.TestUtils.getPropertyValue; - -import java.io.File; -import java.net.URI; - -import org.junit.Test; -import org.springframework.beans.factory.BeanDefinitionStoreException; -import org.springframework.context.support.ClassPathXmlApplicationContext; -import org.springframework.integration.aws.s3.AmazonS3InboundSynchronizationMessageSource; -import org.springframework.integration.aws.s3.core.AmazonS3Operations; -import org.springframework.integration.aws.s3.core.DefaultAmazonS3Operations; -import org.springframework.integration.endpoint.SourcePollingChannelAdapter; - -/** - * The test case class for S3 inbound channel adapter - * - * @author Amol Nayak - * @author Li Wang - * @since 0.5 - * - */ -public class AmazonS3InboundChannelAdapterParserTests { - - - /** - * Tests the inbound channel adapter definition with a valid combination of attributes - */ - @Test - public void withValidAttributeValues() { - ClassPathXmlApplicationContext ctx = new ClassPathXmlApplicationContext("classpath:s3-valid-inbound-cases.xml"); - SourcePollingChannelAdapter valid = ctx.getBean("validInbound", SourcePollingChannelAdapter.class); - AmazonS3InboundSynchronizationMessageSource source = getPropertyValue(valid, "source", AmazonS3InboundSynchronizationMessageSource.class); - assertEquals("TestBucket", getPropertyValue(source, "bucket")); - assertEquals(".temp", getPropertyValue(source, "temporarySuffix")); - assertEquals(new File("/"), getPropertyValue(source, "directory")); - assertEquals("remote", getPropertyValue(source, "remoteDirectory")); - assertEquals(true, getPropertyValue(source, "acceptSubFolders", Boolean.class).booleanValue()); - assertEquals(100, getPropertyValue(source, "maxObjectsPerBatch", Integer.class).intValue()); - assertEquals("[A-Za-z0-9]+\\\\.txt", getPropertyValue(source, "fileNameRegex")); - - //test the second definition with custom attributes - valid = ctx.getBean("validInboundWithCustomOps", SourcePollingChannelAdapter.class); - source = getPropertyValue(valid, "source", AmazonS3InboundSynchronizationMessageSource.class); - assertEquals(new File(System.getProperty("java.io.tmpdir")), getPropertyValue(source, "directory")); - AmazonS3Operations s3Operations = getPropertyValue(source, "s3Operations", AmazonS3Operations.class); - assertEquals(AmazonS3DummyOperations.class, s3Operations.getClass()); - - //test with aws endpoint set - valid = ctx.getBean("withAWSEndpoint", SourcePollingChannelAdapter.class); - source = getPropertyValue(valid, "source", AmazonS3InboundSynchronizationMessageSource.class); - s3Operations = getPropertyValue(source, "s3Operations", AmazonS3Operations.class); - assertEquals(DefaultAmazonS3Operations.class, s3Operations.getClass()); - assertEquals("https://s3-eu-west-1.amazonaws.com", getPropertyValue(s3Operations, "client.endpoint", URI.class).toString()); - - - ctx.close(); - - } - - /** - * Tests with a definition where none of directory and directory-expression attributes are provided - */ - @Test(expected=BeanDefinitionStoreException.class) - public void withNoneOfDirectoryExprAndDirectory() { - new ClassPathXmlApplicationContext("classpath:s3-with-none-of-direxpr-and-dir.xml"); - } -} diff --git a/src/test/java/org/springframework/integration/aws/s3/core/AbstractAmazonS3OperationsImplAWSTests.java b/src/test/java/org/springframework/integration/aws/s3/core/AbstractAmazonS3OperationsImplAWSTests.java deleted file mode 100644 index a48c67f..0000000 --- a/src/test/java/org/springframework/integration/aws/s3/core/AbstractAmazonS3OperationsImplAWSTests.java +++ /dev/null @@ -1,629 +0,0 @@ - /* - * Copyright 2002-2015 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.springframework.integration.aws.s3.core; - -import static org.springframework.integration.aws.common.AWSTestUtils.getCredentials; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FileOutputStream; -import java.io.IOException; -import java.util.Collections; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import org.junit.Assert; - -import org.junit.BeforeClass; -import org.junit.Ignore; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; -import org.springframework.integration.aws.core.AWSCredentials; -import org.springframework.integration.aws.core.PropertiesAWSCredentials; - -import com.amazonaws.auth.BasicAWSCredentials; -import com.amazonaws.services.s3.AmazonS3Client; -import com.amazonaws.services.s3.model.AccessControlList; -import com.amazonaws.services.s3.model.GetObjectRequest; -import com.amazonaws.services.s3.model.Grant; -import com.amazonaws.services.s3.model.ObjectMetadata; -import com.amazonaws.services.s3.model.S3Object; - -/** - * The abstract test class for testing all the common functionality for AWS operations - * on S3 using the appropriate implementation provided by the subclass. - * - * Note: To run the test, you will have to create one bucket for yourself and - * set the name in the {@link #BUCKET_NAME} - * - * @author Amol Nayak - * - * @since 0.5 - * - */ -@Ignore() -public abstract class AbstractAmazonS3OperationsImplAWSTests { - - private static final String VALID_CANONICAL_ID = "f854da004ee08cf4f8664334d288561c8512c508db9785388de7319ded85f8f3"; - - @Rule - public final TemporaryFolder temp = new TemporaryFolder(); - - //private static final String UPLOAD_SOURCE_DIRECTORY = System.getProperty("java.io.tmpdir") + "upload"; - - - //To run the test, you will have to create one one bucket for yourself and - //set the name here - protected static final String BUCKET_NAME = "com.si.aws.test.bucket"; - - private static AmazonS3Client client; - private static PropertiesAWSCredentials credentials; - - @BeforeClass - public static void setup() throws Exception { - AWSCredentials credentials = getCredentials(); - client = new AmazonS3Client( - new BasicAWSCredentials(credentials.getAccessKey(), credentials.getSecretKey())); - } - - - /** - * Sets the multipart threshold value of the upload to 5K, - * this should get executed successfully - */ - @Test - public void withMultipartThresholdWith5k() { - AbstractAmazonS3Operations impl = getS3OperationsImplementation(); - impl.setMultipartUploadThreshold(5120); - Assert.assertEquals(5120,impl.getMultipartUploadThreshold()); - } - - /** - * Sets the multipart threshold value of the upload to a value < 5K, - * should throw IllegalArgumentException - */ - @Test(expected=IllegalArgumentException.class) - public void withMultipartThresholdWithLt5k() { - getS3OperationsImplementation().setMultipartUploadThreshold(5000); - } - - /** - * Sets the directory path as a null value. - */ - @Test(expected=IllegalArgumentException.class) - public void setNullDirectoryString() { - getS3OperationsImplementation().setTemporaryDirectory((String)null); - } - - /** - * Pass a non null string that exists - */ - @Test - public void setNonNullDirectoryString() { - //this will exist - String directory = System.getProperty("java.io.tmpdir"); - getS3OperationsImplementation().setTemporaryDirectory(directory); - } - - /** - * Pass a String to a directory that doesn't exist - */ - @Test(expected=IllegalArgumentException.class) - public void setNonExistentDirectory() { - //getting the current time in millis and hope no folder with that name exists - long current = System.currentTimeMillis(); - DefaultAmazonS3Operations s3Service = new DefaultAmazonS3Operations(credentials); - s3Service.setTemporaryDirectory("./" + current); - } - - /** - * Sets the temporary file suffix to the null value - */ - @Test(expected=IllegalArgumentException.class) - public void setNullTemporaryFileSuffix() { - getS3OperationsImplementation().setTemporaryFileSuffix(null); - } - - /** - * Sets the temporary file suffix to a string that begins with a "." - */ - @Test - public void setValidTempFileSuffixStartingWithDot() { - AbstractAmazonS3Operations impl = getS3OperationsImplementation(); - impl.setTemporaryFileSuffix(".tempsuff"); - Assert.assertEquals(".tempsuff", impl.getTemporaryFileSuffix()); - } - - - /** - * Sets the temporary file suffix to a string that does not begin with a "." - */ - @Test - public void setValidTempFileSuffixStartingWithoutDot() { - AbstractAmazonS3Operations impl = getS3OperationsImplementation(); - impl.setTemporaryFileSuffix("tmpsuff"); - Assert.assertEquals(".tmpsuff", impl.getTemporaryFileSuffix()); - } - - //TODO. Test all the conditions that test the folder generation logic, null folder - //null bucket etc - - /** - * The AWS Service test put a file with null bucket given - * - */ - public void putToNullBucket() { - AbstractAmazonS3Operations impl = getS3OperationsImplementation(); - impl.putObject(null, "/", "name", null); - } - - //TODO: Execute the following cases for putObject - - /** - * Executes the put object with a null bucket name, should throw - * an {@link IllegalArgumentException} - */ - @Test(expected=IllegalArgumentException.class) - public void putWithNullBucket() { - AmazonS3Operations impl = getS3OperationsImplementation(); - impl.putObject(null, null, "SomeObject", null); - } - - /** - * Executes the put object with a null object name, should throw - * an {@link IllegalArgumentException} - */ - @Test(expected=IllegalArgumentException.class) - public void withNullObjectName() { - AmazonS3Operations impl = getS3OperationsImplementation(); - impl.putObject(BUCKET_NAME, "/", null, null); - } - - /** - * Executes the put object with a null s3 object, should throw - * an {@link IllegalArgumentException} - */ - @Test(expected=IllegalArgumentException.class) - public void withNullS3Object() { - AmazonS3Operations impl = getS3OperationsImplementation(); - impl.putObject(BUCKET_NAME, "/", "TestObjectName.txt", null); - } - - /** - * Executes the put object with both file source and input stream provided - * - * an {@link IllegalArgumentException} - */ - @Test(expected=IllegalArgumentException.class) - public void withBothFileSourceAndInputStream() throws Exception { - String folder = temp.getRoot().getAbsolutePath(); - File file = new File(folder + File.separator + "SomeTestFile.txt"); - file.createNewFile(); - FileInputStream fin = new FileInputStream(file); - new AmazonS3Object(null, null,fin, file); - fin.close(); - file.delete(); - } - - /** - * Executes the put object with none of file source and input stream provided - * - * an {@link IllegalArgumentException} - */ - @Test(expected=IllegalArgumentException.class) - public void withNoneOfFileSourceAndInputStream() throws Exception { - new AmazonS3Object(null, null, null, null); - } - - /** - * With a temp file, upload to the bucket and see temp file gets deleted and the object - * successfully uploaded. Also the ACL of the provided object is null, so the - * object should get default ACLs - * - */ - @Test - public void putFromTempFile() throws Exception { - //first delete the file from the AWS bucket - String key = "TestPutFromTempFile.txt"; - deleteObject(BUCKET_NAME, key); - - - File file = generateUploadFile(); - - //Now put the object - AbstractAmazonS3Operations operations = getS3OperationsImplementation(); - operations.setTemporaryDirectory(temp.getRoot()); - FileInputStream fin = new FileInputStream(file); - //No ACL associated - AmazonS3Object object = new AmazonS3Object(null, null,fin, null); - operations.putObject(BUCKET_NAME, null, key, object); - - assertTempFileDeletion(temp.getRoot().getAbsolutePath(), key); - - assertObjectExistenceInBucket(key); - fin.close(); - //delete the source file. - file.delete(); - } - - - /** - * Upload a file to a folder in the given bucket with the folder name - * ending with a slash. Also the ACL of the provided object is null, so the - * object should get default ACLs. - * - */ - @Test - public void putToFolderWithEndingSlash() throws Exception { - String uploadFileName = "TestPutWithEndingSlash.txt"; - String key = "somedir/with/endingslash/" + uploadFileName; - //first delete the file from the AWS bucket - deleteObject(BUCKET_NAME, key); - - File file = generateUploadFile(); - - //Now put the object - AmazonS3Operations operations = getS3OperationsImplementation(); - - //No ACL associated - AmazonS3Object object = new AmazonS3Object(null, null, null, file); - operations.putObject(BUCKET_NAME, "somedir/with/endingslash/", uploadFileName, object); - - - assertObjectExistenceInBucket(key); - //delete the source file. - file.delete(); - } - - /** - * Upload a file to a folder in the given bucket with the folder name - * ending without a slash. Also the ACL of the provided object is null, so the - * object should get default ACLs. - * - */ - @Test - public void putToFolderWithoutEndingSlash() throws Exception { - String uploadFile = "TestPutWithoutEndingSlash.txt"; - String key = "somedir/without/endingslash/" + uploadFile; - //first delete the file from the AWS bucket - deleteObject(BUCKET_NAME, key); - - File file = generateUploadFile(); - - //Now put the object - AmazonS3Operations operations = getS3OperationsImplementation(); - - //No ACL associated - AmazonS3Object object = new AmazonS3Object(null, null, null, file); - operations.putObject(BUCKET_NAME, "somedir/without/endingslash", uploadFile, object); - - assertObjectExistenceInBucket(key); - //delete the source file. - file.delete(); - } - - /** - * Upload a file to a folder in the given bucket with the folder name - * beginning a slash. Also the ACL of the provided object is null, so the - * object should get default ACLs. - * - */ - @Test - public void putToFolderBeginningWithSlash() throws Exception { - String uploadFileName = "TestPutBeginningWithSlash.txt"; - String key = "beginning/with/slash/" + uploadFileName; - //first delete the file from the AWS bucket - deleteObject(BUCKET_NAME, key); - - File file = generateUploadFile(); - //Now put the object - AmazonS3Operations operations = getS3OperationsImplementation(); - - //No ACL associated - AmazonS3Object object = new AmazonS3Object(null, null, null, file); - operations.putObject(BUCKET_NAME, "/beginning/with/slash/", uploadFileName, object); - - assertObjectExistenceInBucket(key); - - //delete the source file. - file.delete(); - } - - - /** - * Upload a file with the provided ACLs and meta data, the test verifies if the - * ACLs and the metadata of the file is appropriately set - * - */ - @Test - public void putToFolderForACLAndMetadataTest() throws Exception { - String uploadFileName = "TestObjectACLAndMetaData.txt"; - String key = "acl/and/metadata/test/" + uploadFileName; - //first delete the file from the AWS bucket - deleteObject(BUCKET_NAME, key); - - File file = generateUploadFile(); - FileInputStream fin = new FileInputStream(file); - - //Now put the object - AmazonS3Operations operations = getS3OperationsImplementation(); - - Map userMetaData = Collections.singletonMap("TestKey", "TestValue"); - AmazonS3ObjectACL acl = new AmazonS3ObjectACL(); - ObjectGrant grant = new ObjectGrant(new Grantee(VALID_CANONICAL_ID, GranteeType.CANONICAL_GRANTEE_TYPE), - ObjectPermissions.READ_ACP); - acl.addGrant(grant); - AmazonS3Object object = new AmazonS3Object(userMetaData, null, fin, null,acl); - operations.putObject(BUCKET_NAME, "/acl/and/metadata/test/", uploadFileName, object); - - //This fails somehow on my machine - //assertTempFileDeletion(uploadFileName); - - //NOTE: The case of the key is no longer in the case we used, its all lower case. - //lets get the object's User metadata first - S3Object s3Object = getObject(BUCKET_NAME,key); - ObjectMetadata objectMetadata = s3Object.getObjectMetadata(); - userMetaData = objectMetadata.getUserMetadata(); - Assert.assertNotNull("User metadata is not expected to be null, but got null", userMetaData); - Assert.assertTrue("Expecting the key 'testkey' in user MetaData", userMetaData.containsKey("testkey")); - Assert.assertEquals("TestValue", userMetaData.get("testkey")); - - //lets verify the object's ACL - AccessControlList acls = getObjectACL(BUCKET_NAME, key); - List grants = acls.getGrantsAsList(); - boolean isACLValid = false; - for(Grant g : grants) { - com.amazonaws.services.s3.model.Grantee grantee = g.getGrantee(); - if(VALID_CANONICAL_ID.equals(grantee.getIdentifier()) - && "READ_ACP".equals(grant.getPermission().toString())) { - isACLValid = true; - } - } - Assert.assertTrue("Expected Object ACl not found", isACLValid); - fin.close(); - - //delete the source file. - file.delete(); - } - - /** - * List the contents in the bucket with null bucket name - */ - @Test(expected=IllegalArgumentException.class) - public void listWithNullBucket() { - AmazonS3Operations impl = getS3OperationsImplementation(); - impl.listObjects(null, "folder", null, 1); - } - - /** - * List objects with negative page size - */ - @Test(expected=IllegalArgumentException.class) - public void listWithPageSizeLt0() { - AmazonS3Operations impl = getS3OperationsImplementation(); - impl.listObjects(BUCKET_NAME, "folder", null, -2); - } - - /** - * List with null folder - */ - @Test - public void listWithNullFolder() { - AmazonS3Operations impl = getS3OperationsImplementation(); - PaginatedObjectsView pov = impl.listObjects(BUCKET_NAME, null, null, 100); - List summary = pov.getObjectSummary(); - Assert.assertNotNull(summary); - Assert.assertTrue(summary.size() > 0); - System.out.println("Summary list size is " + summary.size()); - } - - /** - * List with folder as a slash(/), for root folder - */ - @Test - public void listWithSlashOnRoot() { - AmazonS3Operations impl = getS3OperationsImplementation(); - PaginatedObjectsView pov = impl.listObjects(BUCKET_NAME, "/", null, 100); - List summary = pov.getObjectSummary(); - Assert.assertNotNull(summary); - Assert.assertTrue(summary.size() > 0); - System.out.println("Summary list size is " + summary.size()); - } - - /** - * List with folder as a slash(/) - */ - @Test - public void listWithFolderAsSlash() { - AmazonS3Operations impl = getS3OperationsImplementation(); - PaginatedObjectsView pov = impl.listObjects(BUCKET_NAME, "/acl", null, 100); - List summary = pov.getObjectSummary(); - Assert.assertNotNull(summary); - Assert.assertTrue(summary.size() > 0); - System.out.println("Summary list size is " + summary.size()); - } - - /** - * List with folder as a not beginning with slash(/) - */ - @Test - public void listWithFolderNotBeginningWithSlash() { - AmazonS3Operations impl = getS3OperationsImplementation(); - PaginatedObjectsView pov = impl.listObjects(BUCKET_NAME, "somedir/with", null, 100); - List summary = pov.getObjectSummary(); - Assert.assertNotNull(summary); - Assert.assertTrue(summary.size() > 0); - System.out.println("Summary list size is " + summary.size()); - } - - /** - * The test case assumes that all previous AWS tests are executed and we have at least 4 objects - * in the bucket, on running all the above tests you will have 5, so we need not do anything - * special to add more objects to execute this test - * - */ - @Test - public void paginateRecords() { - AmazonS3Operations impl = getS3OperationsImplementation(); - PaginatedObjectsView pov = impl.listObjects(BUCKET_NAME, "/", null, 3); - List summary = pov.getObjectSummary(); - Assert.assertNotNull(summary); - Assert.assertEquals(3, summary.size()); - String nextMarker = pov.getNextMarker(); - Assert.assertNotNull("Expected a non null marker", nextMarker); - pov = impl.listObjects(BUCKET_NAME, "/", nextMarker, 3); - summary = pov.getObjectSummary(); - Assert.assertNotNull(summary); - Assert.assertTrue(summary.size() > 0); - System.out.printf("Number of records on second page are %d\n",summary.size()); - } - - /** - * Tests the get object with a null bucket - */ - @Test(expected=IllegalArgumentException.class) - public void getObjectFromNullBucket() { - AmazonS3Operations impl = getS3OperationsImplementation(); - impl.getObject(null, null, "ObjectName.txt"); - } - - /** - * Gets a non existent object from the bucket, should return null on unsuccessful search - * and if the object with the key doesn't exist. - */ - @Test - public void getNonExistentObject() { - AmazonS3Operations impl = getS3OperationsImplementation(); - AmazonS3Object object = impl.getObject(BUCKET_NAME, null, "jhgkmjbhdc.thb"); - Assert.assertNull("Expecting a null object but got a non null one", object); - } - - /** - * Invoked the getObject with null folder, this will get the object - * from the root of the bucket. - * - */ - @Test - public void getObjectWithNullFolder() { - AmazonS3Operations impl = getS3OperationsImplementation(); - AmazonS3Object object = impl.getObject(BUCKET_NAME, null, "TestPutFromTempFile.txt"); - Assert.assertNotNull("Expecting a non null object but got a null one", object); - - } - - /** - * Invoked the getObject with folder name beginning with / - * - */ - @Test - public void getObjectromFolderBeginningWithSlash() { - AmazonS3Operations impl = getS3OperationsImplementation(); - AmazonS3Object object = impl.getObject(BUCKET_NAME, "/acl/and/metadata/test", "TestObjectACLAndMetaData.txt"); - Assert.assertNotNull("Expecting a non null object but got a null one", object); - } - - /** - * Invoked the getObject with folder name beginning with / - * - */ - @Test - public void getObjectFromFolderBeginningWithoutSlash() { - AmazonS3Operations impl = getS3OperationsImplementation(); - AmazonS3Object object = impl.getObject(BUCKET_NAME, "acl/and/metadata/test", "TestObjectACLAndMetaData.txt"); - Assert.assertNotNull("Expecting a non null object but got a null one", object); - } - - - /** - * The common method that checks if the temp file generated for the - * test file uploaded is deleted. - */ - private void assertTempFileDeletion(String rootFolder, String baseFileName) { - //Check if the temp file exists. - File tempFile = new File(rootFolder + File.separator + baseFileName + ".writing"); - Assert.assertFalse("Was expecting the temp file to be deleted, but is present", tempFile.exists()); - } - - /** - * Common method that will assert the existence of the object with the given key in the - * bucket - * - * @param key - */ - private void assertObjectExistenceInBucket(String key) { - S3Object s3Object = getObject(BUCKET_NAME, key); - //This is not needed as an exception will be thrown if the key does not exist - Assert.assertNotNull("Non null S3Object expected",s3Object); - } - - /** - * The private helper method that generates the test file to be uploaded - * @throws IOException - * @throws FileNotFoundException - */ - private File generateUploadFile() throws IOException, FileNotFoundException { - //TODO: Move this to @BeforeClass? - String fileName = System.currentTimeMillis() + ".txt"; - File file = new File(temp.getRoot().getAbsolutePath() + File.separator + fileName); - file.createNewFile(); - //Write something to it - FileOutputStream fos = new FileOutputStream(file); - fos.write("Test".getBytes()); - fos.close(); - return file; - } - - - - //-- helper methods to interact with AWS S3 services - //These methods are used to verify and assert if the implementation - //has performed the desired operation - /** - * Gets the object from the S3 bucket - * - * @param gets the object from the bucket with the given key - */ - protected S3Object getObject(String bucket,String key) { - return client.getObject(new GetObjectRequest(bucket, key)); - } - - /** - * Gets the objects's {@link AccessControlList} (ACL) for the given bucket and key - * - * @param bucket - * @param key - */ - protected AccessControlList getObjectACL(String bucket,String key) { - return client.getObjectAcl(bucket, key); - } - - /** - * Deleted the given object name from the given bucket and key - * - * @param bucket - * @param key - * - */ - protected void deleteObject(String bucket,String key) { - client.deleteObject(bucket, key); - } - - - protected abstract AbstractAmazonS3Operations getS3OperationsImplementation(); -} diff --git a/src/test/java/org/springframework/integration/aws/s3/core/DefaultAmazonS3OperationsAWSTests.java b/src/test/java/org/springframework/integration/aws/s3/core/DefaultAmazonS3OperationsAWSTests.java deleted file mode 100644 index dccf043..0000000 --- a/src/test/java/org/springframework/integration/aws/s3/core/DefaultAmazonS3OperationsAWSTests.java +++ /dev/null @@ -1,87 +0,0 @@ - /* - * Copyright 2002-2013 the original author or authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - package org.springframework.integration.aws.s3.core; - -import java.util.concurrent.Executors; -import java.util.concurrent.ThreadPoolExecutor; - -import org.junit.Assert; - -import org.junit.BeforeClass; -import org.junit.Ignore; -import org.junit.Test; -import org.springframework.integration.aws.core.PropertiesAWSCredentials; - -/** - * The test class for the {@link DefaultAmazonS3Operations}, the default implementation that - * uses the AWS SDK to implement the functionality. The tests are present in the superclass - * {@link AbstractAmazonS3OperationsImplAWSTests} - * - * Please note that that this test needs connectivity with the AWS S3 service - * to be successfully executed. It is excluded from the maven's test execution by default - * - * To run this test you need to have your AWSAccess key and Secret key in the - * file awscredentials.properties in the classpath. This file is not present in the - * repository and you need to add one yourselves to src/test/resources folder and have - * two properties accessKey and secretKey in it containing the access and the secret key - * - * - * @author Amol Nayak - * - * @since 0.5 - * - */ -@Ignore -public class DefaultAmazonS3OperationsAWSTests extends AbstractAmazonS3OperationsImplAWSTests { - - - private static DefaultAmazonS3Operations impl; - - @BeforeClass - public static void setupS3Operations() throws Exception { - PropertiesAWSCredentials credentials = - new PropertiesAWSCredentials("classpath:awscredentials.properties"); - credentials.afterPropertiesSet(); - impl = new DefaultAmazonS3Operations(credentials); - } - /** - * Sets the thread pool executor to a non null value, execution should - * complete successfully - * - */ - @Ignore - @Test - public void withNonNullThreadPoolExecutor() { - ThreadPoolExecutor executor = (ThreadPoolExecutor)Executors.newFixedThreadPool(10); - impl.setThreadPoolExecutor(executor); - Assert.assertEquals(executor, impl.getThreadPoolExecutor()); - } - - /** - * Sets the thread pool executor to a null value, should throw an - * {@link IllegalArgumentException} - */ - @Ignore - @Test(expected=IllegalArgumentException.class) - public void withNullThreadPoolExecutor() { - impl.setThreadPoolExecutor(null); - } - - @Override - protected AbstractAmazonS3Operations getS3OperationsImplementation() { - return impl; - } -} diff --git a/src/test/resources/s3-both-customfilegenerator-and-expression.xml b/src/test/resources/s3-both-customfilegenerator-and-expression.xml deleted file mode 100644 index f7abd8d..0000000 --- a/src/test/resources/s3-both-customfilegenerator-and-expression.xml +++ /dev/null @@ -1,27 +0,0 @@ - - - - - - - - - - diff --git a/src/test/resources/s3-custom-operations-with-disallowed-attributes.xml b/src/test/resources/s3-custom-operations-with-disallowed-attributes.xml deleted file mode 100644 index 79bb33c..0000000 --- a/src/test/resources/s3-custom-operations-with-disallowed-attributes.xml +++ /dev/null @@ -1,28 +0,0 @@ - - - - - - - - - - \ No newline at end of file diff --git a/src/test/resources/s3-multiupload-lessthan-5120.xml b/src/test/resources/s3-multiupload-lessthan-5120.xml deleted file mode 100644 index 0c2753e..0000000 --- a/src/test/resources/s3-multiupload-lessthan-5120.xml +++ /dev/null @@ -1,21 +0,0 @@ - - - - - - - diff --git a/src/test/resources/s3-valid-inbound-cases.xml b/src/test/resources/s3-valid-inbound-cases.xml deleted file mode 100644 index bb910ce..0000000 --- a/src/test/resources/s3-valid-inbound-cases.xml +++ /dev/null @@ -1,72 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/src/test/resources/s3-valid-outbound-cases.xml b/src/test/resources/s3-valid-outbound-cases.xml deleted file mode 100644 index 2d4847e..0000000 --- a/src/test/resources/s3-valid-outbound-cases.xml +++ /dev/null @@ -1,104 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/src/test/resources/s3-with-none-of-direxpr-and-dir.xml b/src/test/resources/s3-with-none-of-direxpr-and-dir.xml deleted file mode 100644 index 7720825..0000000 --- a/src/test/resources/s3-with-none-of-direxpr-and-dir.xml +++ /dev/null @@ -1,26 +0,0 @@ - - - - - - - - - - \ No newline at end of file diff --git a/src/test/resources/testawscredentials.properties b/src/test/resources/testawscredentials.properties deleted file mode 100644 index 89f7ecb..0000000 --- a/src/test/resources/testawscredentials.properties +++ /dev/null @@ -1,2 +0,0 @@ -accessKey=dummy -secretKey=dummy \ No newline at end of file