From c21533e42c4f352b9db3346a511a6ec046b5e7bf Mon Sep 17 00:00:00 2001 From: Timo Salm <52704922+tsalm-pivotal@users.noreply.github.com> Date: Tue, 7 Jul 2020 20:27:31 +0200 Subject: [PATCH] Support for s3 compatible storage * Add support for s3 compatible storage to s3 consumer and supplier. * Add enhancements to s3 compatible storage implementation based on code review * Move s3 compatible storage endpoint configuration bean to AmazonS3Configuration to remove code duplication * Disable auto configuration of AmazonS3 Bean to enable custom configurations * Add documentation for endpoint url configuration property * Use localhost instead of a real URL to avoid an unexpected connection to the internet --- applications/sink/s3-sink/README.adoc | 1 + ...onfiguration-metadata-whitelist.properties | 3 +- applications/source/s3-source/README.adoc | 1 + ...onfiguration-metadata-whitelist.properties | 3 +- functions/common/aws-s3-common/pom.xml | 12 +++ .../fn/common/aws/s3/AmazonS3Properties.java | 39 ++++++++++ ...ompatibleStorageAmazonS3Configuration.java | 51 ++++++++++++ .../main/resources/META-INF/spring.factories | 1 + .../src/main/resources/application.properties | 1 + .../aws/s3/AmazonS3ConfigurationTests.java | 78 +++++++++++++++++++ functions/consumer/s3-consumer/README.adoc | 4 +- functions/supplier/s3-supplier/README.adoc | 7 +- 12 files changed, 195 insertions(+), 6 deletions(-) create mode 100644 functions/common/aws-s3-common/src/main/java/org/springframework/cloud/fn/common/aws/s3/AmazonS3Properties.java create mode 100644 functions/common/aws-s3-common/src/main/java/org/springframework/cloud/fn/common/aws/s3/CompatibleStorageAmazonS3Configuration.java create mode 100644 functions/common/aws-s3-common/src/main/resources/application.properties create mode 100644 functions/common/aws-s3-common/src/test/java/org/springframework/cloud/fn/common/aws/s3/AmazonS3ConfigurationTests.java diff --git a/applications/sink/s3-sink/README.adoc b/applications/sink/s3-sink/README.adoc index 449e42ee..95010fd6 100644 --- a/applications/sink/s3-sink/README.adoc +++ b/applications/sink/s3-sink/README.adoc @@ -20,6 +20,7 @@ $$s3.consumer.acl-expression$$:: $$Expression to evaluate S3 Object access contr $$s3.consumer.bucket$$:: $$AWS bucket for target file(s) to store.$$ *($$String$$, default: `$$$$`)* $$s3.consumer.bucket-expression$$:: $$Expression to evaluate AWS bucket name.$$ *($$Expression$$, default: `$$$$`)* $$s3.consumer.key-expression$$:: $$Expression to evaluate S3 Object key.$$ *($$Expression$$, default: `$$$$`)* +$$s3.common.endpoint-url$$:: $$Optional endpoint url to connect to s3 compatible storage.$$ *($$String$$, default: `$$$$`)* //end::configuration-properties[] The target generated application based on the `AmazonS3SinkConfiguration` can be enhanced with the `S3MessageHandler.UploadMetadataProvider` and/or `S3ProgressListener`, which are injected into `S3MessageHandler` bean. diff --git a/applications/sink/s3-sink/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties b/applications/sink/s3-sink/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties index 29e031a9..caea643c 100644 --- a/applications/sink/s3-sink/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties +++ b/applications/sink/s3-sink/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties @@ -1 +1,2 @@ -configuration-properties.classes=org.springframework.cloud.fn.consumer.s3.AwsS3ConsumerProperties +configuration-properties.classes=org.springframework.cloud.fn.consumer.s3.AwsS3ConsumerProperties,\ + org.springframework.cloud.fn.common.aws.s3.AmazonS3Properties diff --git a/applications/source/s3-source/README.adoc b/applications/source/s3-source/README.adoc index dca73d61..73fca7e3 100644 --- a/applications/source/s3-source/README.adoc +++ b/applications/source/s3-source/README.adoc @@ -62,6 +62,7 @@ $$s3.supplier.preserve-timestamp$$:: $$To transfer or not the timestamp of the r $$s3.supplier.remote-dir$$:: $$AWS S3 bucket resource.$$ *($$String$$, default: `$$bucket$$`)* $$s3.supplier.remote-file-separator$$:: $$Remote File separator.$$ *($$String$$, default: `$$/$$`)* $$s3.supplier.tmp-file-suffix$$:: $$Temporary file suffix.$$ *($$String$$, default: `$$.tmp$$`)* +$$s3.common.endpoint-url$$:: $$Optional endpoint url to connect to s3 compatible storage.$$ *($$String$$, default: `$$$$`)* //end::configuration-properties[] == Amazon AWS common options diff --git a/applications/source/s3-source/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties b/applications/source/s3-source/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties index b5f88127..d87af164 100644 --- a/applications/source/s3-source/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties +++ b/applications/source/s3-source/src/main/resources/META-INF/dataflow-configuration-metadata-whitelist.properties @@ -1,2 +1,3 @@ configuration-properties.classes=org.springframework.cloud.fn.supplier.s3.AwsS3SupplierProperties,\ - org.springframework.cloud.fn.common.file.FileConsumerProperties + org.springframework.cloud.fn.common.file.FileConsumerProperties,\ + org.springframework.cloud.fn.common.aws.s3.AmazonS3Properties diff --git a/functions/common/aws-s3-common/pom.xml b/functions/common/aws-s3-common/pom.xml index 71784a07..768f6899 100644 --- a/functions/common/aws-s3-common/pom.xml +++ b/functions/common/aws-s3-common/pom.xml @@ -33,6 +33,18 @@ org.springframework.integration spring-integration-file + + + org.springframework.boot + spring-boot-starter-test + test + + + org.junit.vintage + junit-vintage-engine + + + diff --git a/functions/common/aws-s3-common/src/main/java/org/springframework/cloud/fn/common/aws/s3/AmazonS3Properties.java b/functions/common/aws-s3-common/src/main/java/org/springframework/cloud/fn/common/aws/s3/AmazonS3Properties.java new file mode 100644 index 00000000..6409eb5e --- /dev/null +++ b/functions/common/aws-s3-common/src/main/java/org/springframework/cloud/fn/common/aws/s3/AmazonS3Properties.java @@ -0,0 +1,39 @@ +/* + * Copyright 2020-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.common.aws.s3; + +import org.springframework.boot.context.properties.ConfigurationProperties; + +/** + * @author Timo Salm + */ +@ConfigurationProperties("s3.common") +public class AmazonS3Properties { + + /** + * Optional endpoint url to connect to s3 compatible storage. + */ + private String endpointUrl; + + public String getEndpointUrl() { + return this.endpointUrl; + } + + public void setEndpointUrl(String endpointUrl) { + this.endpointUrl = endpointUrl; + } +} diff --git a/functions/common/aws-s3-common/src/main/java/org/springframework/cloud/fn/common/aws/s3/CompatibleStorageAmazonS3Configuration.java b/functions/common/aws-s3-common/src/main/java/org/springframework/cloud/fn/common/aws/s3/CompatibleStorageAmazonS3Configuration.java new file mode 100644 index 00000000..cdbdab91 --- /dev/null +++ b/functions/common/aws-s3-common/src/main/java/org/springframework/cloud/fn/common/aws/s3/CompatibleStorageAmazonS3Configuration.java @@ -0,0 +1,51 @@ +/* + * Copyright 2020-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.common.aws.s3; + +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.client.builder.AwsClientBuilder.EndpointConfiguration; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3ClientBuilder; + +import org.springframework.boot.autoconfigure.AutoConfigureBefore; +import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.cloud.aws.core.region.RegionProvider; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +/** + * @author Timo Salm + */ +@Configuration +@EnableConfigurationProperties(AmazonS3Properties.class) +@AutoConfigureBefore(AmazonS3Configuration.class) +public class CompatibleStorageAmazonS3Configuration { + + @Bean + @ConditionalOnProperty("s3.common.endpoint-url") + public AmazonS3 compatibleStorageAmazonS3(AWSCredentialsProvider awsCredentialsProvider, RegionProvider regionProvider, + AmazonS3Properties amazonS3Properties) { + final AmazonS3ClientBuilder builder = AmazonS3ClientBuilder.standard(); + final EndpointConfiguration endpointConfiguration = new EndpointConfiguration( + amazonS3Properties.getEndpointUrl(), regionProvider.getRegion().getName()); + builder.setEndpointConfiguration(endpointConfiguration); + return builder + .withCredentials(awsCredentialsProvider) + .build(); + } +} diff --git a/functions/common/aws-s3-common/src/main/resources/META-INF/spring.factories b/functions/common/aws-s3-common/src/main/resources/META-INF/spring.factories index ed864b17..b6e0ea51 100644 --- a/functions/common/aws-s3-common/src/main/resources/META-INF/spring.factories +++ b/functions/common/aws-s3-common/src/main/resources/META-INF/spring.factories @@ -1,2 +1,3 @@ org.springframework.boot.autoconfigure.EnableAutoConfiguration=\ + org.springframework.cloud.fn.common.aws.s3.CompatibleStorageAmazonS3Configuration,\ org.springframework.cloud.fn.common.aws.s3.AmazonS3Configuration diff --git a/functions/common/aws-s3-common/src/main/resources/application.properties b/functions/common/aws-s3-common/src/main/resources/application.properties new file mode 100644 index 00000000..64d4b5fe --- /dev/null +++ b/functions/common/aws-s3-common/src/main/resources/application.properties @@ -0,0 +1 @@ +spring.autoconfigure.exclude=org.springframework.cloud.aws.autoconfigure.context.ContextResourceLoaderAutoConfiguration diff --git a/functions/common/aws-s3-common/src/test/java/org/springframework/cloud/fn/common/aws/s3/AmazonS3ConfigurationTests.java b/functions/common/aws-s3-common/src/test/java/org/springframework/cloud/fn/common/aws/s3/AmazonS3ConfigurationTests.java new file mode 100644 index 00000000..834d8770 --- /dev/null +++ b/functions/common/aws-s3-common/src/test/java/org/springframework/cloud/fn/common/aws/s3/AmazonS3ConfigurationTests.java @@ -0,0 +1,78 @@ +/* + * Copyright 2020-2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.fn.common.aws.s3; + +import com.amazonaws.auth.AWSCredentialsProvider; +import com.amazonaws.auth.AWSStaticCredentialsProvider; +import com.amazonaws.auth.BasicAWSCredentials; +import com.amazonaws.services.s3.AmazonS3; +import com.amazonaws.services.s3.AmazonS3Client; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import org.springframework.boot.context.annotation.UserConfigurations; +import org.springframework.boot.test.context.runner.ApplicationContextRunner; +import org.springframework.cloud.aws.core.region.RegionProvider; +import org.springframework.cloud.aws.core.region.StaticRegionProvider; +import org.springframework.context.annotation.Bean; + +/** + * @author Timo Salm + */ +public class AmazonS3ConfigurationTests { + + private final ApplicationContextRunner runner = new ApplicationContextRunner() + .withConfiguration(UserConfigurations.of(CompatibleStorageAmazonS3Configuration.class, + AmazonS3Configuration.class, TestConfiguration.class)); + + private final String testRegionName = "eu-central-1"; + + @Test + public void testAmazonS3Configuration() { + runner.withPropertyValues().run(context -> { + final AmazonS3Client amazonS3 = (AmazonS3Client) context.getBean(AmazonS3.class); + Assertions.assertNotNull(amazonS3); + Assertions.assertEquals(testRegionName, amazonS3.getRegionName()); + Assertions.assertTrue(amazonS3.getResourceUrl("b", "k") + .startsWith("https://s3.eu-central-1.amazonaws.com")); + }); + } + + @Test + public void testAmazonS3ConfigurationForS3CompatibleStorage() { + runner.withPropertyValues( + "s3.common.endpoint-url=http://localhost:8080" + ).run(context -> { + final AmazonS3Client amazonS3 = (AmazonS3Client) context.getBean(AmazonS3.class); + Assertions.assertNotNull(amazonS3); + Assertions.assertTrue(amazonS3.getResourceUrl("b", "k") + .startsWith("http://localhost:8080")); + }); + } + + private static class TestConfiguration { + @Bean + RegionProvider regionProvider() { + return new StaticRegionProvider("eu-central-1"); + } + + @Bean + AWSCredentialsProvider awsCredentialsProvider() { + return new AWSStaticCredentialsProvider(new BasicAWSCredentials("accessKey", "secretKey")); + } + } +} diff --git a/functions/consumer/s3-consumer/README.adoc b/functions/consumer/s3-consumer/README.adoc index 9339f5da..cc24f84c 100644 --- a/functions/consumer/s3-consumer/README.adoc +++ b/functions/consumer/s3-consumer/README.adoc @@ -14,8 +14,10 @@ You can use `s3Consumer` as a qualifier when injecting. ## Configuration Options All configuration properties are prefixed with `s3.consumer`. +There are also properties that need to be used with the prefix `s3.common`. -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/consumer/s3/AwsS3ConsumerProperties.java[AwsS3ConsumerProperties]. +For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/consumer/s3/AwsS3ConsumerProperties.java[AwsS3ConsumerProperties] and +link:../../common/aws-s3-common/src/main/java/org/springframework/cloud/fn/common/aws/s3/AmazonS3Properties.java[AmazonS3Properties]. ## Examples diff --git a/functions/supplier/s3-supplier/README.adoc b/functions/supplier/s3-supplier/README.adoc index 86a2cd24..c01ccc61 100644 --- a/functions/supplier/s3-supplier/README.adoc +++ b/functions/supplier/s3-supplier/README.adoc @@ -21,10 +21,11 @@ Once injected, you can use the `get` method of the `Supplier` to invoke it and t ## Configuration Options All configuration properties are prefixed with `s3.supplier`. -There are also properties that need to be used with the prefix `file.consumer`. +There are also properties that need to be used with the prefix `s3.common` and `file.consumer`. -For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/supplier/s3/AwsS3SupplierProperties.java[AwsS3upplierProperties]. -See link:../../common/file-common/src/main/java/org/springframework/cloud/fn/common/file/FileConsumerProperties.java[this] also. +For more information on the various options available, please see link:src/main/java/org/springframework/cloud/fn/supplier/s3/AwsS3SupplierProperties.java[AwsS3upplierProperties], +link:../../common/file-common/src/main/java/org/springframework/cloud/fn/common/file/FileConsumerProperties.java[FileConsumerProperties], and +link:../../common/aws-s3-common/src/main/java/org/springframework/cloud/fn/common/aws/s3/AmazonS3Properties.java[AmazonS3Properties]. ## Tests