SpringCloud/spring-cloud-config
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch '4.1.x' into 4.2.x

Browse Source
This commit is contained in:
Ryan Baxter
2025-04-21 19:33:48 -04:00
parent f6da12c541 d13647e52b
commit 80aa935ca9
6 changed files with 388 additions and 114 deletions
Download Patch File Download Diff File Expand all files Collapse all files

448
spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepository.java
View File

@@ -16,13 +16,13 @@
package org.springframework.cloud.config.server.environment;
import java.io.IOException;
import java.io.InputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Properties;
import java.util.function.Consumer;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -31,6 +31,7 @@ import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
import org.springframework.beans.factory.config.YamlProcessor;
import org.springframework.beans.factory.config.YamlPropertiesFactoryBean;
import org.springframework.cloud.config.environment.Environment;
import org.springframework.cloud.config.environment.PropertySource;
@@ -40,6 +41,8 @@ import org.springframework.core.io.InputStreamResource;
import org.springframework.util.ObjectUtils;
import org.springframework.util.StringUtils;
import static org.springframework.cloud.config.server.environment.AwsS3EnvironmentRepository.PATH_SEPARATOR;
/**
* @author Clay McCoy
* @author Scott Frederick
@@ -47,12 +50,12 @@ import org.springframework.util.StringUtils;
*/
public class AwsS3EnvironmentRepository implements EnvironmentRepository, Ordered, SearchPathLocator {
protected static final String PATH_SEPARATOR = "/";
private static final Log LOG = LogFactory.getLog(AwsS3EnvironmentRepository.class);
private static final String AWS_S3_RESOURCE_SCHEME = "s3://";
private static final String PATH_SEPARATOR = "/";
private final S3Client s3Client;
private final String bucketName;
@@ -86,7 +89,6 @@ public class AwsS3EnvironmentRepository implements EnvironmentRepository, Ordere
String[] profileArray = parseProfiles(profiles);
List<String> apps = Arrays.asList(StringUtils.commaDelimitedListToStringArray(application.replace(" ", "")));
Collections.reverse(apps);
if (!apps.contains(serverProperties.getDefaultApplicationName())) {
apps = new ArrayList<>(apps);
apps.add(serverProperties.getDefaultApplicationName());
@@ -95,29 +97,7 @@ public class AwsS3EnvironmentRepository implements EnvironmentRepository, Ordere
final Environment environment = new Environment(application, profileArray);
environment.setLabel(label);
List<String> labels;
if (StringUtils.hasText(label) && label.contains(",")) {
labels = Arrays.asList(StringUtils.commaDelimitedListToStringArray(label));
Collections.reverse(labels);
}
else {
labels = Collections.singletonList(label);
}
for (String l : labels) {
for (String profile : profileArray) {
for (String app : apps) {
addPropertySource(environment, app, profile, l);
}
}
}
// Add propertysources without profiles as well
for (String l : labels) {
for (String app : apps) {
addPropertySource(environment, app, null, l);
}
}
addPropertySources(environment, apps, profileArray, label);
if (LOG.isDebugEnabled()) {
LOG.debug("Returning Environment: " + environment);
@@ -126,22 +106,69 @@ public class AwsS3EnvironmentRepository implements EnvironmentRepository, Ordere
return environment;
}
private void addPropertySource(Environment environment, String app, String profile, String label) {
S3ConfigFile s3ConfigFile = getS3ConfigFile(app, profile, label);
if (s3ConfigFile != null) {
environment.setVersion(s3ConfigFile.getVersion());
private void addPropertySources(Environment environment, List<String> apps, String[] profiles, String label) {
// If we have profiles, add property sources with those profiles
for (String profile : profiles) {
addPropertySourcesForApps(apps, app -> addProfileSpecificPropertySource(environment, app, profile, label));
}
// If we have no profiles just add property sources for all apps
if (profiles.length == 0) {
addPropertySourcesForApps(apps, app -> addNonProfileSpecificPropertySource(environment, app, null, label));
}
else {
// If we have profiles, we still need to add property sources from files that
// are not profile specific but we pass
// along the profiles as well so we can check if any non-profile specific YAML
// files have profile specific documents
// within them
for (String profile : profiles) {
addPropertySourcesForApps(apps,
app -> addNonProfileSpecificPropertySource(environment, app, profile, label));
}
}
}
private void addPropertySourcesForApps(List<String> apps, Consumer<String> addPropertySource) {
apps.forEach(addPropertySource);
}
private void addProfileSpecificPropertySource(Environment environment, String app, String profile, String label) {
List<S3ConfigFile> s3ConfigFiles = getS3ConfigFile(app, profile, label, this::getS3PropertiesOrJsonConfigFile,
this::getProfileSpecificS3ConfigFileYaml);
addPropertySource(environment, s3ConfigFiles);
}
private void addNonProfileSpecificPropertySource(Environment environment, String app, String profile,
String label) {
List<S3ConfigFile> s3ConfigFiles = getS3ConfigFile(app, profile, label,
this::getNonProfileSpecificPropertiesOrJsonConfigFile, this::getNonProfileSpecificS3ConfigFileYaml);
addPropertySource(environment, s3ConfigFiles);
}
private void addPropertySource(Environment environment, List<S3ConfigFile> s3ConfigFiles) {
for (S3ConfigFile s3ConfigFile : s3ConfigFiles) {
final Properties config = s3ConfigFile.read();
config.putAll(serverProperties.getOverrides());
StringBuilder propertySourceName = new StringBuilder().append("s3:").append(app);
if (profile != null) {
propertySourceName.append("-").append(profile);
// This logic handles the case where the s3 file is a YAML file that is
// not profile specific (ie it does not have -<profile> in the name)
// and does not have any profile specific documents in it. In this case we do
// not want to include this
// property source we only want to include the document for the default
// profile. When we create
// the S3ConfigFile for this file we set the
// shouldIncludeWithEmptyProperties to false
// in ProfileSpecificYamlDocumentS3ConfigFile for this specific case.
if (config != null) {
if (!config.isEmpty() || s3ConfigFile.isShouldIncludeWithEmptyProperties()) {
environment.setVersion(s3ConfigFile.getVersion());
config.putAll(serverProperties.getOverrides());
PropertySource propertySource = new PropertySource(s3ConfigFile.getName(), config);
if (LOG.isDebugEnabled()) {
LOG.debug("Adding property source to environment " + propertySource);
}
environment.add(propertySource);
}
}
PropertySource propertySource = new PropertySource(propertySourceName.toString(), config);
if (LOG.isDebugEnabled()) {
LOG.debug("Adding property source to environment " + propertySource);
}
environment.add(propertySource);
}
}
@@ -149,73 +176,89 @@ public class AwsS3EnvironmentRepository implements EnvironmentRepository, Ordere
return StringUtils.commaDelimitedListToStringArray(profiles);
}
private S3ConfigFile getS3ConfigFile(String application, String profile, String label) {
String objectKeyPrefix = buildObjectKeyPrefix(application, profile, label);
return getS3ConfigFile(objectKeyPrefix);
private List<S3ConfigFile> getS3ConfigFile(String application, String profile, String label,
JsonOrPropertiesS3ConfigFileCreator creator, YamlS3ConfigFileCreator yamlCreator) {
S3ConfigFile configFile = creator.create(application, profile, label);
if (configFile != null) {
return List.of(configFile);
}
return new ArrayList<>(yamlCreator.create(application, profile, label));
}
private String buildObjectKeyPrefix(String application, String profile, String label) {
StringBuilder objectKeyPrefix = new StringBuilder();
if (!ObjectUtils.isEmpty(label)) {
objectKeyPrefix.append(label).append(PATH_SEPARATOR);
}
objectKeyPrefix.append(application);
if (!ObjectUtils.isEmpty(profile)) {
objectKeyPrefix.append("-").append(profile);
}
return objectKeyPrefix.toString();
}
private S3ConfigFile getS3ConfigFile(String keyPrefix) {
if (LOG.isDebugEnabled()) {
LOG.debug("Getting S3 config file for prefix " + keyPrefix);
private List<YamlS3ConfigFile> getNonProfileSpecificS3ConfigFileYaml(String application, String profile,
String label) {
List<YamlS3ConfigFile> configFiles = new ArrayList<>();
if (profile != null) {
try {
YamlS3ConfigFile configFileDocument = new ProfileSpecificYamlDocumentS3ConfigFile(application, profile,
label, bucketName, s3Client);
configFileDocument.setShouldIncludeWithEmptyProperties(false);
configFiles.add(configFileDocument);
}
catch (Exception e) {
if (LOG.isDebugEnabled()) {
LOG.debug("Did not find specific yaml document in non-profile specific file using application <"
+ application + "> profile <" + profile + "> label <" + label + ">.");
}
}
}
try {
final ResponseInputStream<GetObjectResponse> responseInputStream = getObject(keyPrefix + ".properties");
return new PropertyS3ConfigFile(responseInputStream.response().versionId(), responseInputStream);
YamlS3ConfigFile configFile = new NonProfileSpecificYamlDocumentS3ConfigFile(application, null, label,
bucketName, s3Client);
configFiles.add(configFile);
}
catch (Exception eProperties) {
try {
if (LOG.isDebugEnabled()) {
LOG.debug("Did not find " + keyPrefix + ".properties. Trying yml extension", eProperties);
}
final ResponseInputStream<GetObjectResponse> responseInputStream = getObject(keyPrefix + ".yml");
return new YamlS3ConfigFile(responseInputStream.response().versionId(), responseInputStream);
}
catch (Exception eYml) {
try {
if (LOG.isDebugEnabled()) {
LOG.debug("Did not find " + keyPrefix + ".yml. Trying yaml extension", eYml);
}
final ResponseInputStream<GetObjectResponse> responseInputStream = getObject(keyPrefix + ".yaml");
return new YamlS3ConfigFile(responseInputStream.response().versionId(), responseInputStream);
}
catch (Exception eYaml) {
try {
if (LOG.isDebugEnabled()) {
LOG.debug("Did not find " + keyPrefix + ".yaml. Trying json extension", eYaml);
}
final ResponseInputStream<GetObjectResponse> responseInputStream = getObject(
keyPrefix + ".json");
return new JsonS3ConfigFile(responseInputStream.response().versionId(), responseInputStream);
}
catch (Exception eJson) {
if (LOG.isDebugEnabled()) {
LOG.debug("Did not find S3 config file with properties, yml, yaml, or json extension for "
+ keyPrefix, eJson);
}
return null;
}
}
catch (Exception e) {
if (LOG.isDebugEnabled()) {
LOG.debug(
"Did not find non-profile specific yaml document in non-profile specific file using application <"
+ application + "> profile <null>" + " label <" + label + ">.");
}
}
return configFiles;
}
private List<YamlS3ConfigFile> getProfileSpecificS3ConfigFileYaml(String application, String profile,
String label) {
try {
YamlS3ConfigFile configFile = new ProfileSpecificYamlS3ConfigFile(application, profile, label, bucketName,
s3Client);
return List.of(configFile);
}
catch (Exception e) {
LOG.warn("Could not read YAML file", e);
return Collections.emptyList();
}
}
private ResponseInputStream<GetObjectResponse> getObject(String key) throws Exception {
private S3ConfigFile getNonProfileSpecificPropertiesOrJsonConfigFile(String application, String profile,
String label) {
return getS3PropertiesOrJsonConfigFile(application, null, label);
}
private S3ConfigFile getS3PropertiesOrJsonConfigFile(String application, String profile, String label) {
if (LOG.isDebugEnabled()) {
LOG.debug("Getting object with key " + key);
LOG.debug("Getting S3 config file for " + application + " " + profile + " " + label);
}
try {
return new PropertyS3ConfigFile(application, profile, label, bucketName, s3Client);
}
catch (Exception propertyException) {
if (LOG.isDebugEnabled()) {
LOG.debug("Did not find properties file using application <" + application + "> profile <" + profile
+ "> label <" + label + ">. Trying json extension", propertyException);
}
try {
return new JsonS3ConfigFile(application, profile, label, bucketName, s3Client);
}
catch (Exception jsonException) {
if (LOG.isDebugEnabled()) {
LOG.debug("Did not find json file using application <" + application + "> profile <" + profile
+ "> label <" + label + ">. Trying yaml extension", jsonException);
}
return null;
}
}
return s3Client.getObject(GetObjectRequest.builder().bucket(bucketName).key(key).build());
}
@Override
@@ -232,16 +275,46 @@ public class AwsS3EnvironmentRepository implements EnvironmentRepository, Ordere
return new Locations(application, profiles, label, null, new String[] { baseLocation.toString() });
}
interface YamlS3ConfigFileCreator {
List<YamlS3ConfigFile> create(String application, String profile, String label);
}
interface JsonOrPropertiesS3ConfigFileCreator {
S3ConfigFile create(String application, String profile, String label);
}
}
abstract class S3ConfigFile {
protected static final Log LOG = LogFactory.getLog(S3ConfigFile.class);
private final String version;
protected String application;
protected S3ConfigFile(String version) {
this.version = version;
protected String label;
protected String profile;
protected String bucketName;
protected S3Client s3Client;
protected Properties properties;
private String version;
private boolean shouldIncludeWithEmptyProperties = true;
protected S3ConfigFile(String application, String profile, String label, String bucketName, S3Client s3Client) {
this.application = application;
this.profile = profile;
this.label = label;
this.bucketName = bucketName;
this.s3Client = s3Client;
}
String getVersion() {
@@ -250,62 +323,209 @@ abstract class S3ConfigFile {
abstract Properties read();
boolean isShouldIncludeWithEmptyProperties() {
return shouldIncludeWithEmptyProperties;
}
void setShouldIncludeWithEmptyProperties(boolean shouldIncludeWithEmptyProperties) {
this.shouldIncludeWithEmptyProperties = shouldIncludeWithEmptyProperties;
}
public String getName() {
return createPropertySourceName(application, profile);
}
protected ResponseInputStream<GetObjectResponse> getObject() throws Exception {
assert (getExtensions() != null && !getExtensions().isEmpty());
List<String> extensions = getExtensions();
for (int i = 0; i < extensions.size(); i++) {
String key = buildObjectKeyPrefix() + "." + extensions.get(i);
if (LOG.isDebugEnabled()) {
LOG.debug("Getting object with key " + key);
}
try {
ResponseInputStream<GetObjectResponse> inputStream = s3Client
.getObject(GetObjectRequest.builder().bucket(bucketName).key(key).build());
this.version = inputStream.response().versionId();
return inputStream;
}
catch (Exception e) {
if (i < extensions.size() - 1) {
if (LOG.isDebugEnabled()) {
LOG.debug("Did not find " + key + ". Trying next extension", e);
}
}
else {
throw e;
}
}
}
return null;
}
protected abstract List<String> getExtensions();
protected String buildObjectKeyPrefix() {
return buildObjectKeyPrefix(true);
}
String buildObjectKeyPrefix(boolean includeProfile) {
StringBuilder objectKeyPrefix = new StringBuilder();
if (!ObjectUtils.isEmpty(label)) {
objectKeyPrefix.append(label).append(PATH_SEPARATOR);
}
objectKeyPrefix.append(application);
if (!ObjectUtils.isEmpty(profile) && includeProfile) {
objectKeyPrefix.append("-").append(profile);
}
return objectKeyPrefix.toString();
}
private String createPropertySourceName(String app, String profile) {
StringBuilder propertySourceName = new StringBuilder().append("s3:").append(app);
if (profile != null) {
propertySourceName.append("-").append(profile);
}
return propertySourceName.toString();
}
}
class PropertyS3ConfigFile extends S3ConfigFile {
final InputStream inputStream;
PropertyS3ConfigFile(String version, InputStream inputStream) {
super(version);
this.inputStream = inputStream;
PropertyS3ConfigFile(String application, String profile, String label, String bucketName, S3Client s3Client) {
super(application, profile, label, bucketName, s3Client);
this.properties = read();
}
@Override
public Properties read() {
if (this.properties != null) {
return this.properties;
}
Properties props = new Properties();
try (InputStream in = inputStream) {
try (InputStream in = getObject()) {
props.load(in);
}
catch (IOException e) {
catch (Exception e) {
LOG.warn("Exception thrown when reading property file", e);
throw new IllegalStateException("Cannot load environment", e);
}
return props;
}
@Override
protected List<String> getExtensions() {
return List.of("properties");
}
}
class YamlS3ConfigFile extends S3ConfigFile {
final InputStream inputStream;
final YamlProcessor.DocumentMatcher[] documentMatchers;
YamlS3ConfigFile(String version, InputStream inputStream) {
super(version);
this.inputStream = inputStream;
YamlS3ConfigFile(String application, String profile, String label, String bucketName, S3Client s3Client) {
this(application, profile, label, bucketName, s3Client, new YamlProcessor.DocumentMatcher[] {});
}
YamlS3ConfigFile(String application, String profile, String label, String bucketName, S3Client s3Client,
final YamlProcessor.DocumentMatcher... documentMatchers) {
super(application, profile, label, bucketName, s3Client);
this.documentMatchers = documentMatchers;
this.properties = read();
}
protected static boolean profileMatchesActivateProperty(String profile, Properties properties) {
return profile.equals(properties.get("spring.config.activate.on-profile"))
|| profile.equals(properties.get("spring.config.activate.onProfile"));
}
protected static boolean onProfilePropertyExists(Properties properties) {
return properties.get("spring.config.activate.on-profile") != null
|| properties.get("spring.config.activate.onProfile") != null;
}
@Override
public Properties read() {
if (properties != null) {
return properties;
}
final YamlPropertiesFactoryBean yaml = new YamlPropertiesFactoryBean();
try (InputStream in = inputStream) {
try (InputStream in = getObject()) {
yaml.setResources(new InputStreamResource(in));
yaml.setDocumentMatchers(documentMatchers);
return yaml.getObject();
}
catch (IOException e) {
catch (Exception e) {
LOG.warn("Could not read YAML file", e);
throw new IllegalStateException("Cannot load environment", e);
}
}
@Override
protected List<String> getExtensions() {
return List.of("yml", "yaml");
}
}
class ProfileSpecificYamlDocumentS3ConfigFile extends YamlS3ConfigFile {
ProfileSpecificYamlDocumentS3ConfigFile(String application, String profile, String label, String bucketName,
S3Client s3Client) {
super(application, profile, label, bucketName, s3Client,
properties -> profileMatchesActivateProperty(profile, properties) ? YamlProcessor.MatchStatus.FOUND
: YamlProcessor.MatchStatus.NOT_FOUND);
}
@Override
public boolean isShouldIncludeWithEmptyProperties() {
return false;
}
@Override
protected String buildObjectKeyPrefix() {
return super.buildObjectKeyPrefix(false);
}
}
class NonProfileSpecificYamlDocumentS3ConfigFile extends YamlS3ConfigFile {
NonProfileSpecificYamlDocumentS3ConfigFile(String application, String profile, String label, String bucketName,
S3Client s3Client) {
super(application, profile, label, bucketName, s3Client, properties -> !onProfilePropertyExists(properties)
? YamlProcessor.MatchStatus.FOUND : YamlProcessor.MatchStatus.NOT_FOUND);
}
}
class ProfileSpecificYamlS3ConfigFile extends YamlS3ConfigFile {
ProfileSpecificYamlS3ConfigFile(String application, String profile, String label, String bucketName,
S3Client s3Client) {
super(application, profile, label, bucketName, s3Client,
properties -> !onProfilePropertyExists(properties) ? YamlProcessor.MatchStatus.ABSTAIN
: profileMatchesActivateProperty(profile, properties) ? YamlProcessor.MatchStatus.FOUND
: YamlProcessor.MatchStatus.NOT_FOUND);
}
}
class JsonS3ConfigFile extends YamlS3ConfigFile {
// YAML is a superset of JSON, which means you can parse JSON with a YAML parser
JsonS3ConfigFile(String version, InputStream inputStream) {
super(version, inputStream);
JsonS3ConfigFile(String application, String profile, String label, String bucketName, S3Client s3Client) {
super(application, profile, label, bucketName, s3Client);
this.properties = read();
}
@Override
protected List<String> getExtensions() {
return List.of("json");
}
}

38
spring-cloud-config-server/src/test/java/org/springframework/cloud/config/server/environment/AwsS3EnvironmentRepositoryTests.java
View File

@@ -16,7 +16,10 @@
package org.springframework.cloud.config.server.environment;
import java.io.IOException;
import java.io.UnsupportedEncodingException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
@@ -44,6 +47,8 @@ import software.amazon.awssdk.services.s3.model.VersioningConfiguration;
import org.springframework.cloud.config.environment.Environment;
import org.springframework.cloud.config.environment.PropertySource;
import org.springframework.cloud.config.server.config.ConfigServerProperties;
import org.springframework.core.io.ClassPathResource;
import org.springframework.core.io.Resource;
import static org.assertj.core.api.Assertions.assertThat;
import static org.testcontainers.containers.localstack.LocalStackContainer.Service.S3;
@@ -127,6 +132,39 @@ public class AwsS3EnvironmentRepositoryTests {
toBeRemoved.clear();
}
@Test
public void multiDocumentYaml() throws IOException {
Resource resource = new ClassPathResource("awss3/foo.yaml");
String yamlString = new String(Files.readAllBytes(Paths.get(resource.getURI())));
putFiles("foo.yaml", yamlString);
resource = new ClassPathResource("awss3/foo-test1.yaml");
yamlString = new String(Files.readAllBytes(Paths.get(resource.getURI())));
putFiles("foo-test1.yaml", yamlString);
resource = new ClassPathResource("awss3/application-test1.yaml");
yamlString = new String(Files.readAllBytes(Paths.get(resource.getURI())));
putFiles("application-test1.yaml", yamlString);
resource = new ClassPathResource("awss3/application.yaml");
yamlString = new String(Files.readAllBytes(Paths.get(resource.getURI())));
putFiles("application.yaml", yamlString);
final Environment env = envRepo.findOne("foo", "test1", null);
assertThat(env.getPropertySources().size()).isEqualTo(5);
List<PropertySource> propertySources = env.getPropertySources();
// @formatter:off
assertThat(propertySources.get(0).getName()).isEqualTo("s3:foo-test1"); // foo-test1.yaml
assertThat(propertySources.get(0).getSource().get("app")).isEqualTo("test-test1-yaml");
assertThat(propertySources.get(1).getName()).isEqualTo("s3:application-test1"); // application-test1.yaml
assertThat(propertySources.get(1).getSource().get("app")).isEqualTo("test1-yaml");
assertThat(propertySources.get(2).getName()).isEqualTo("s3:foo-test1"); // profile specific document in foo.yaml
assertThat(propertySources.get(2).getSource().get("a")).isEqualTo(1);
assertThat(propertySources.get(2).getSource().get("spring.config.activate.onProfile")).isEqualTo("test1");
assertThat(propertySources.get(3).getName()).isEqualTo("s3:foo"); // non-profile specific document in foo.yaml
assertThat(propertySources.get(3).getSource().get("a")).isEqualTo(0);
assertThat(propertySources.get(4).getName()).isEqualTo("s3:application"); // application.yaml
assertThat(propertySources.get(4).getSource().get("app")).isEqualTo("yaml");
// @formatter:on
}
@Test
public void failToFindNonexistentObject() {
Environment env = envRepo.findOne("foo", "bar", null);

1
spring-cloud-config-server/src/test/resources/awss3/application-test1.yaml Normal file
View File

@@ -0,0 +1 @@
app: test1-yaml

1
spring-cloud-config-server/src/test/resources/awss3/application.yaml Normal file
View File

@@ -0,0 +1 @@
app: yaml

1
spring-cloud-config-server/src/test/resources/awss3/foo-test1.yaml Normal file
View File

@@ -0,0 +1 @@
app: test-test1-yaml

13
spring-cloud-config-server/src/test/resources/awss3/foo.yaml Normal file
View File

@@ -0,0 +1,13 @@
a: 0
---
spring:
config:
activate:
onProfile: test1
a: 1
---
spring:
config:
activate:
on-profile: test2
a: 2