Add Resource API support to AWS S3 EnvironmentRepository (#1517)
* Clean up AWS s3 EnvironmentRepository * Don't require a profile prefix for the default profile - either 'application.[properties|yml|json]' or 'application-default.[properties|yml|json]' will match. * Support a comma-delimited list of profiles and load each specified profile. * Add Resource API support to the AwsS3EnvironmentRepository. Fixes #1506
This commit is contained in:
Scott Frederick
committed by
Spencer Gibb
Spencer Gibb
parent
6bd1b5a37b
commit
04fb5618d6
@@ -18,7 +18,6 @@ package org.springframework.cloud.config.server.environment;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.InputStream;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
|
||||
import com.amazonaws.services.s3.AmazonS3;
|
||||
@@ -36,8 +35,13 @@ import org.springframework.util.StringUtils;
|
||||
|
||||
/**
|
||||
* @author Clay McCoy
|
||||
* @author Scott Frederick
|
||||
*/
|
||||
public class AwsS3EnvironmentRepository implements EnvironmentRepository, Ordered {
|
||||
public class AwsS3EnvironmentRepository
|
||||
implements EnvironmentRepository, Ordered, SearchPathLocator {
|
||||
|
||||
private static final String AWS_S3_RESOURCE_SCHEME = "s3://";
|
||||
private static final String PATH_SEPARATOR = "/";
|
||||
|
||||
private final AmazonS3 s3Client;
|
||||
|
||||
@@ -64,39 +68,68 @@ public class AwsS3EnvironmentRepository implements EnvironmentRepository, Ordere
|
||||
}
|
||||
|
||||
@Override
|
||||
public Environment findOne(String specifiedApplication, String specifiedProfile,
|
||||
public Environment findOne(String specifiedApplication, String specifiedProfiles,
|
||||
String specifiedLabel) {
|
||||
final String application = StringUtils.isEmpty(specifiedApplication)
|
||||
? serverProperties.getDefaultApplicationName() : specifiedApplication;
|
||||
final String profile = StringUtils.isEmpty(specifiedProfile)
|
||||
? serverProperties.getDefaultProfile() : specifiedProfile;
|
||||
final String profiles = StringUtils.isEmpty(specifiedProfiles)
|
||||
? serverProperties.getDefaultProfile() : specifiedProfiles;
|
||||
final String label = StringUtils.isEmpty(specifiedLabel)
|
||||
? serverProperties.getDefaultLabel() : specifiedLabel;
|
||||
StringBuilder objectKeyPrefix = new StringBuilder();
|
||||
if (!StringUtils.isEmpty(label)) {
|
||||
objectKeyPrefix.append(label).append("/");
|
||||
}
|
||||
objectKeyPrefix.append(application).append("-").append(profile);
|
||||
final Environment environment = new Environment(application, profile);
|
||||
|
||||
String[] profileArray = parseProfiles(profiles);
|
||||
|
||||
final Environment environment = new Environment(application, profileArray);
|
||||
environment.setLabel(label);
|
||||
|
||||
for (String profile : profileArray) {
|
||||
S3ConfigFile s3ConfigFile = getS3ConfigFile(application, profile, label);
|
||||
if (s3ConfigFile != null) {
|
||||
environment.setVersion(s3ConfigFile.getVersion());
|
||||
|
||||
final Properties config = s3ConfigFile.read();
|
||||
config.putAll(serverProperties.getOverrides());
|
||||
StringBuilder propertySourceName = new StringBuilder().append("s3:")
|
||||
.append(application);
|
||||
if (profile != null) {
|
||||
propertySourceName.append("-").append(profile);
|
||||
}
|
||||
environment
|
||||
.add(new PropertySource(propertySourceName.toString(), config));
|
||||
}
|
||||
}
|
||||
|
||||
return environment;
|
||||
}
|
||||
|
||||
private String[] parseProfiles(String profiles) {
|
||||
if (profiles.equals(serverProperties.getDefaultProfile())) {
|
||||
return new String[] { profiles, null };
|
||||
}
|
||||
return StringUtils.commaDelimitedListToStringArray(profiles);
|
||||
}
|
||||
|
||||
private S3ConfigFile getS3ConfigFile(String application, String profile,
|
||||
String label) {
|
||||
String objectKeyPrefix = buildObjectKeyPrefix(application, profile, label);
|
||||
|
||||
final S3ObjectIdBuilder s3ObjectIdBuilder = new S3ObjectIdBuilder()
|
||||
.withBucket(bucketName);
|
||||
S3ConfigFile s3ConfigFile = getS3ConfigFile(s3ObjectIdBuilder,
|
||||
objectKeyPrefix.toString());
|
||||
if (s3ConfigFile == null) {
|
||||
throw new NoSuchRepositoryException(
|
||||
"No such repository: ("
|
||||
+ s3ObjectIdBuilder
|
||||
.withKey(objectKeyPrefix.toString()
|
||||
+ "(.properties | .yml | .json)")
|
||||
.build().toString()
|
||||
+ ")");
|
||||
|
||||
return getS3ConfigFile(s3ObjectIdBuilder, objectKeyPrefix);
|
||||
}
|
||||
|
||||
private String buildObjectKeyPrefix(String application, String profile,
|
||||
String label) {
|
||||
StringBuilder objectKeyPrefix = new StringBuilder();
|
||||
if (!StringUtils.isEmpty(label)) {
|
||||
objectKeyPrefix.append(label).append(PATH_SEPARATOR);
|
||||
}
|
||||
environment.setVersion(s3ConfigFile.getVersion());
|
||||
final Map config = s3ConfigFile.read();
|
||||
config.putAll(serverProperties.getOverrides());
|
||||
environment.add(new PropertySource(application, config));
|
||||
return environment;
|
||||
objectKeyPrefix.append(application);
|
||||
if (!StringUtils.isEmpty(profile)) {
|
||||
objectKeyPrefix.append("-").append(profile);
|
||||
}
|
||||
return objectKeyPrefix.toString();
|
||||
}
|
||||
|
||||
private S3ConfigFile getS3ConfigFile(S3ObjectIdBuilder s3ObjectIdBuilder,
|
||||
@@ -128,6 +161,15 @@ public class AwsS3EnvironmentRepository implements EnvironmentRepository, Ordere
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Locations getLocations(String application, String profiles, String label) {
|
||||
String baseLocation = AWS_S3_RESOURCE_SCHEME + bucketName + PATH_SEPARATOR
|
||||
+ application;
|
||||
|
||||
return new Locations(application, profiles, label, null,
|
||||
new String[] { baseLocation });
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
abstract class S3ConfigFile {
|
||||
@@ -142,7 +184,7 @@ abstract class S3ConfigFile {
|
||||
return version;
|
||||
}
|
||||
|
||||
abstract Map<?, ?> read();
|
||||
abstract Properties read();
|
||||
|
||||
}
|
||||
|
||||
@@ -156,7 +198,7 @@ class PropertyS3ConfigFile extends S3ConfigFile {
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<?, ?> read() {
|
||||
public Properties read() {
|
||||
Properties props = new Properties();
|
||||
try (InputStream in = inputStream) {
|
||||
props.load(in);
|
||||
@@ -179,10 +221,10 @@ class YamlS3ConfigFile extends S3ConfigFile {
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<?, ?> read() {
|
||||
public Properties read() {
|
||||
final YamlPropertiesFactoryBean yaml = new YamlPropertiesFactoryBean();
|
||||
try (InputStream in = inputStream) {
|
||||
yaml.setResources(new InputStreamResource(inputStream));
|
||||
yaml.setResources(new InputStreamResource(in));
|
||||
return yaml.getObject();
|
||||
}
|
||||
catch (IOException e) {
|
||||
|
||||
@@ -30,10 +30,10 @@ import org.junit.Test;
|
||||
import org.mockito.ArgumentMatcher;
|
||||
|
||||
import org.springframework.cloud.config.environment.Environment;
|
||||
import org.springframework.cloud.config.environment.PropertySource;
|
||||
import org.springframework.cloud.config.server.config.ConfigServerProperties;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.assertj.core.api.Assertions.catchThrowable;
|
||||
import static org.mockito.ArgumentMatchers.argThat;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
@@ -50,8 +50,6 @@ public class AwsS3EnvironmentRepositoryTests {
|
||||
final EnvironmentRepository envRepo = new AwsS3EnvironmentRepository(s3Client,
|
||||
"bucket1", server);
|
||||
|
||||
final S3Object s3Object = new S3Object();
|
||||
|
||||
final String propertyContent = "cloudfoundry.enabled=true\n"
|
||||
+ "cloudfoundry.accounts[0].name=acc1\n"
|
||||
+ "cloudfoundry.accounts[0].user=user1\n"
|
||||
@@ -97,138 +95,133 @@ public class AwsS3EnvironmentRepositoryTests {
|
||||
}
|
||||
|
||||
@Test
|
||||
public void failToFindNonexistantObject() {
|
||||
|
||||
Throwable thrown = catchThrowable(() -> {
|
||||
envRepo.findOne("foo", "bar", null);
|
||||
});
|
||||
|
||||
assertThat(thrown).isInstanceOf(NoSuchRepositoryException.class);
|
||||
assertThat(thrown).hasMessage(
|
||||
"No such repository: (bucket: bucket1, key: foo-bar(.properties | .yml | .json), versionId: null)");
|
||||
public void failToFindNonexistentObject() {
|
||||
Environment env = envRepo.findOne("foo", "bar", null);
|
||||
assertThat(env.getPropertySources().size()).isEqualTo(0);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void findPropertiesObject() throws UnsupportedEncodingException {
|
||||
final S3ObjectId S3ObjectId = new S3ObjectId("bucket1", "foo-bar.properties");
|
||||
final GetObjectRequest request = new GetObjectRequest(S3ObjectId);
|
||||
s3Object.setObjectContent(new StringInputStream(propertyContent));
|
||||
when(s3Client.getObject(argThat(new GetObjectRequestMatcher(request))))
|
||||
.thenReturn(s3Object);
|
||||
setupS3("foo-bar.properties", propertyContent);
|
||||
|
||||
// Pulling content from a .properties file forces a boolean into a String
|
||||
expectedProperties.put("cloudfoundry.enabled", "true");
|
||||
|
||||
final Environment env = envRepo.findOne("foo", "bar", null);
|
||||
|
||||
assertThat(env.getName()).isEqualTo("foo");
|
||||
assertThat(env.getProfiles()).isEqualTo(new String[] { "bar" });
|
||||
assertThat(env.getLabel()).isEqualTo(null);
|
||||
assertThat(env.getVersion()).isEqualTo(null);
|
||||
assertThat(env.getPropertySources().size()).isEqualTo(1);
|
||||
assertThat(env.getPropertySources().get(0).getSource())
|
||||
.isEqualTo(expectedProperties);
|
||||
assertExpectedEnvironment(env, "foo", null, null, 1, "bar");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void findJsonObject() throws UnsupportedEncodingException {
|
||||
final S3ObjectId s3ObjectId = new S3ObjectId("bucket1", "foo-bar.json");
|
||||
final GetObjectRequest request = new GetObjectRequest(s3ObjectId);
|
||||
s3Object.setObjectContent(new StringInputStream(jsonContent));
|
||||
when(s3Client.getObject(argThat(new GetObjectRequestMatcher(request))))
|
||||
.thenReturn(s3Object);
|
||||
setupS3("foo-bar.json", jsonContent);
|
||||
|
||||
final Environment env = envRepo.findOne("foo", "bar", null);
|
||||
|
||||
assertThat(env.getName()).isEqualTo("foo");
|
||||
assertThat(env.getProfiles()).isEqualTo(new String[] { "bar" });
|
||||
assertThat(env.getLabel()).isEqualTo(null);
|
||||
assertThat(env.getVersion()).isEqualTo(null);
|
||||
assertThat(env.getPropertySources().size()).isEqualTo(1);
|
||||
assertThat(env.getPropertySources().get(0).getSource())
|
||||
.isEqualTo(expectedProperties);
|
||||
assertExpectedEnvironment(env, "foo", null, null, 1, "bar");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void findYamlObject() throws UnsupportedEncodingException {
|
||||
final S3ObjectId s3ObjectId = new S3ObjectId("bucket1", "foo-bar.yml");
|
||||
final GetObjectRequest request = new GetObjectRequest(s3ObjectId);
|
||||
s3Object.setObjectContent(new StringInputStream(yamlContent));
|
||||
when(s3Client.getObject(argThat(new GetObjectRequestMatcher(request))))
|
||||
.thenReturn(s3Object);
|
||||
setupS3("foo-bar.yml", yamlContent);
|
||||
|
||||
final Environment env = envRepo.findOne("foo", "bar", null);
|
||||
|
||||
assertThat(env.getName()).isEqualTo("foo");
|
||||
assertThat(env.getProfiles()).isEqualTo(new String[] { "bar" });
|
||||
assertThat(env.getLabel()).isEqualTo(null);
|
||||
assertThat(env.getVersion()).isEqualTo(null);
|
||||
assertThat(env.getPropertySources().size()).isEqualTo(1);
|
||||
assertThat(env.getPropertySources().get(0).getSource())
|
||||
.isEqualTo(expectedProperties);
|
||||
assertExpectedEnvironment(env, "foo", null, null, 1, "bar");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void findDefaultProfileObject() throws UnsupportedEncodingException {
|
||||
final S3ObjectId s3ObjectId = new S3ObjectId("bucket1", "foo-default.yml");
|
||||
final GetObjectRequest request = new GetObjectRequest(s3ObjectId);
|
||||
s3Object.setObjectContent(new StringInputStream(yamlContent));
|
||||
when(s3Client.getObject(argThat(new GetObjectRequestMatcher(request))))
|
||||
.thenReturn(s3Object);
|
||||
public void findWithDefaultProfile() throws UnsupportedEncodingException {
|
||||
setupS3("foo.yml", yamlContent);
|
||||
|
||||
final Environment env = envRepo.findOne("foo", null, null);
|
||||
|
||||
assertThat(env.getName()).isEqualTo("foo");
|
||||
assertThat(env.getProfiles()).isEqualTo(new String[] { "default" });
|
||||
assertThat(env.getLabel()).isEqualTo(null);
|
||||
assertThat(env.getVersion()).isEqualTo(null);
|
||||
assertThat(env.getPropertySources().size()).isEqualTo(1);
|
||||
assertThat(env.getPropertySources().get(0).getSource())
|
||||
.isEqualTo(expectedProperties);
|
||||
assertExpectedEnvironment(env, "foo", null, null, 1, "default", null);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void findLabeledObject() throws UnsupportedEncodingException {
|
||||
final S3ObjectId s3ObjectId = new S3ObjectId("bucket1", "label1/foo-bar.yml");
|
||||
final GetObjectRequest request = new GetObjectRequest(s3ObjectId);
|
||||
s3Object.setObjectContent(new StringInputStream(yamlContent));
|
||||
when(s3Client.getObject(argThat(new GetObjectRequestMatcher(request))))
|
||||
.thenReturn(s3Object);
|
||||
public void findWithDefaultProfileUsingSuffix() throws UnsupportedEncodingException {
|
||||
setupS3("foo-default.yml", yamlContent);
|
||||
|
||||
final Environment env = envRepo.findOne("foo", null, null);
|
||||
|
||||
assertExpectedEnvironment(env, "foo", null, null, 1, "default", null);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void findWithMultipleProfilesAllFound() throws UnsupportedEncodingException {
|
||||
setupS3("foo-profile1.yml", yamlContent);
|
||||
setupS3("foo-profile2.yml", jsonContent);
|
||||
|
||||
final Environment env = envRepo.findOne("foo", "profile1,profile2", null);
|
||||
|
||||
assertExpectedEnvironment(env, "foo", null, null, 2, "profile1", "profile2");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void findWithMultipleProfilesOneFound() throws UnsupportedEncodingException {
|
||||
setupS3("foo-profile2.yml", jsonContent);
|
||||
|
||||
final Environment env = envRepo.findOne("foo", "profile1,profile2", null);
|
||||
|
||||
assertExpectedEnvironment(env, "foo", null, null, 1, "profile1", "profile2");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void findWithLabel() throws UnsupportedEncodingException {
|
||||
setupS3("label1/foo-bar.yml", yamlContent);
|
||||
|
||||
final Environment env = envRepo.findOne("foo", "bar", "label1");
|
||||
|
||||
assertThat(env.getName()).isEqualTo("foo");
|
||||
assertThat(env.getProfiles()).isEqualTo(new String[] { "bar" });
|
||||
assertThat(env.getLabel()).isEqualTo("label1");
|
||||
assertThat(env.getVersion()).isEqualTo(null);
|
||||
assertThat(env.getPropertySources().size()).isEqualTo(1);
|
||||
assertThat(env.getPropertySources().get(0).getSource())
|
||||
.isEqualTo(expectedProperties);
|
||||
assertExpectedEnvironment(env, "foo", "label1", null, 1, "bar");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void findVersionedObject() throws UnsupportedEncodingException {
|
||||
final S3ObjectId s3ObjectId = new S3ObjectId("bucket1", "foo-bar.yml");
|
||||
final GetObjectRequest request = new GetObjectRequest(s3ObjectId);
|
||||
s3Object.setObjectContent(new StringInputStream(yamlContent));
|
||||
final ObjectMetadata metadata = new ObjectMetadata();
|
||||
metadata.setHeader("x-amz-version-id", "v1");
|
||||
s3Object.setObjectMetadata(metadata);
|
||||
when(s3Client.getObject(argThat(new GetObjectRequestMatcher(request))))
|
||||
.thenReturn(s3Object);
|
||||
public void findWithVersion() throws UnsupportedEncodingException {
|
||||
setupS3("foo-bar.yml", "v1", yamlContent);
|
||||
|
||||
final Environment env = envRepo.findOne("foo", "bar", null);
|
||||
|
||||
assertThat(env.getName()).isEqualTo("foo");
|
||||
assertThat(env.getProfiles()).isEqualTo(new String[] { "bar" });
|
||||
assertThat(env.getLabel()).isEqualTo(null);
|
||||
assertThat(env.getVersion()).isEqualTo("v1");
|
||||
assertThat(env.getPropertySources().size()).isEqualTo(1);
|
||||
assertThat(env.getPropertySources().get(0).getSource())
|
||||
.isEqualTo(expectedProperties);
|
||||
assertExpectedEnvironment(env, "foo", null, "v1", 1, "bar");
|
||||
}
|
||||
|
||||
private class GetObjectRequestMatcher implements ArgumentMatcher<GetObjectRequest> {
|
||||
private void setupS3(String fileName, String propertyContent)
|
||||
throws UnsupportedEncodingException {
|
||||
setupS3(fileName, null, propertyContent);
|
||||
}
|
||||
|
||||
private void setupS3(String fileName, String version, String propertyContent)
|
||||
throws UnsupportedEncodingException {
|
||||
final S3ObjectId s3ObjectId = new S3ObjectId("bucket1", fileName);
|
||||
final GetObjectRequest request = new GetObjectRequest(s3ObjectId);
|
||||
|
||||
final S3Object s3Object = new S3Object();
|
||||
s3Object.setObjectContent(new StringInputStream(propertyContent));
|
||||
|
||||
if (version != null) {
|
||||
final ObjectMetadata metadata = new ObjectMetadata();
|
||||
metadata.setHeader("x-amz-version-id", version);
|
||||
s3Object.setObjectMetadata(metadata);
|
||||
}
|
||||
|
||||
when(s3Client.getObject(argThat(new GetObjectRequestMatcher(request))))
|
||||
.thenReturn(s3Object);
|
||||
}
|
||||
|
||||
private void assertExpectedEnvironment(Environment env, String applicationName,
|
||||
String label, String version, int propertySourceCount, String... profiles) {
|
||||
assertThat(env.getName()).isEqualTo(applicationName);
|
||||
assertThat(env.getProfiles()).isEqualTo(profiles);
|
||||
assertThat(env.getLabel()).isEqualTo(label);
|
||||
assertThat(env.getVersion()).isEqualTo(version);
|
||||
assertThat(env.getPropertySources().size()).isEqualTo(propertySourceCount);
|
||||
for (PropertySource ps : env.getPropertySources()) {
|
||||
assertThat(ps.getSource()).isEqualTo(expectedProperties);
|
||||
}
|
||||
}
|
||||
|
||||
private static class GetObjectRequestMatcher
|
||||
implements ArgumentMatcher<GetObjectRequest> {
|
||||
|
||||
private final GetObjectRequest expected;
|
||||
|
||||
@@ -238,7 +231,7 @@ public class AwsS3EnvironmentRepositoryTests {
|
||||
|
||||
@Override
|
||||
public boolean matches(GetObjectRequest actual) {
|
||||
if (!(actual instanceof GetObjectRequest)) {
|
||||
if (actual == null) {
|
||||
return false;
|
||||
}
|
||||
return Objects.equals(actual.getBucketName(), expected.getBucketName())
|
||||
|
||||
Reference in New Issue
Block a user