Some cleanup for config server 3 (#1617)

This commit is contained in:
erabii
2024-04-02 23:11:08 +03:00
committed by GitHub
parent 4194a22304
commit b6300eb427
3 changed files with 618 additions and 18 deletions

View File

@@ -23,7 +23,10 @@ import java.util.HashMap;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.function.BiPredicate;
import java.util.function.BooleanSupplier;
import java.util.function.Function;
import java.util.function.Supplier;
import java.util.stream.Collectors;
@@ -51,6 +54,12 @@ public final class ConfigUtils {
private static final Log LOG = LogFactory.getLog(ConfigUtils.class);
// sourceName (configmap or secret name) ends with : "-dev.yaml" or the like.
private static final BiPredicate<String, String> ENDS_WITH_PROFILE_AND_EXTENSION = (sourceName,
activeProfile) -> sourceName.endsWith("-" + activeProfile + ".yml")
|| sourceName.endsWith("-" + activeProfile + ".yaml")
|| sourceName.endsWith("-" + activeProfile + ".properties");
private ConfigUtils() {
}
@@ -205,19 +214,13 @@ public final class ConfigUtils {
rawData = decodeData(rawData);
}
// In some cases we want to include properties from the default profile
// along with any active profiles
// In these cases includeDefaultProfileData will be true
// If includeDefaultProfileData is false then we want to make sure that we
// only return properties from any active profiles
// Check the source to see if it contains any active profiles
boolean containsActiveProfile = environment.getActiveProfiles().length == 0
|| Arrays.stream(environment.getActiveProfiles())
.anyMatch(activeProfile -> sourceName.endsWith("-" + activeProfile)
|| "default".equals(activeProfile));
if (includeDefaultProfileData || containsActiveProfile
|| containsDataWithProfile(rawData, environment.getActiveProfiles())) {
/*
* In some cases we want to include properties from the default profile
* along with any active profiles In these cases includeDefaultProfileData
* will be true If includeDefaultProfileData is false then we want to make
* sure that we only return properties from any active profiles
*/
if (processSource(includeDefaultProfileData, environment, sourceName, rawData)) {
data.putAll(SourceDataEntriesProcessor.processAllEntries(rawData == null ? Map.of() : rawData,
environment, includeDefaultProfileData));
}
@@ -227,13 +230,34 @@ public final class ConfigUtils {
return new MultipleSourcesContainer(foundSourceNames, data);
}
static boolean processSource(boolean includeDefaultProfileData, Environment environment, String sourceName,
Map<String, String> sourceRawData) {
List<String> activeProfiles = Arrays.stream(environment.getActiveProfiles()).toList();
boolean emptyActiveProfiles = activeProfiles.isEmpty();
boolean profileBasedSourceName = activeProfiles.stream()
.anyMatch(activeProfile -> sourceName.endsWith("-" + activeProfile));
boolean defaultProfilePresent = activeProfiles.contains("default");
return includeDefaultProfileData || emptyActiveProfiles || profileBasedSourceName || defaultProfilePresent
|| rawDataContainsProfileBasedSource(activeProfiles, sourceRawData).getAsBoolean();
}
/*
* In the case there the data contains yaml or properties files we need to check their
* names to see if they contain any active profiles.
* this one is not inlined into 'processSource' because other filters, that come
* before it, might have already resolved to 'true', so no need to compute it at all.
*
* This method is supposed to answer the question if raw data that a certain source
* (configmap or secret) has entries that are themselves profile based
* yaml/yml/properties. For example: 'account-k8s.yaml' or the like.
*/
private static boolean containsDataWithProfile(Map<String, String> rawData, String[] activeProfiles) {
return rawData.keySet().stream().anyMatch(key -> Arrays.stream(activeProfiles)
.anyMatch(activeProfile -> key.contains("-" + activeProfile) || "default".equals(activeProfile)));
static BooleanSupplier rawDataContainsProfileBasedSource(List<String> activeProfiles,
Map<String, String> sourceRawData) {
return () -> Optional.ofNullable(sourceRawData).orElse(Map.of()).keySet().stream()
.anyMatch(keyName -> activeProfiles.stream()
.anyMatch(activeProfile -> ENDS_WITH_PROFILE_AND_EXTENSION.test(keyName, activeProfile)));
}
/**

View File

@@ -0,0 +1,481 @@
/*
* Copyright 2013-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.kubernetes.commons.config;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.springframework.boot.test.system.CapturedOutput;
import org.springframework.boot.test.system.OutputCaptureExtension;
import org.springframework.core.env.Environment;
import org.springframework.mock.env.MockEnvironment;
/**
* Class that is supposed to test only ConfigUtils::processSource and
* ConfigUtils::processNamedData.
*
* @author wind57
*/
@ExtendWith(OutputCaptureExtension.class)
class ConfigUtilsProcessSourceTests {
/**
* <pre>
* - includeDefaultProfileData = true
* - emptyActiveProfiles = does not matter
* - profileBasedSourceName = does not matter
* - defaultProfilePresent = does not matter
* - rawDataContainsProfileBasedSource = does not matter
* </pre>
*
* Since 'includeDefaultProfileData=true', all other arguments are irrelevant and
* method must return 'true'.
*/
@Test
void testProcessSourceOne() {
boolean includeDefaultProfileData = true;
Environment environment = new MockEnvironment();
String sourceName = "account";
Map<String, String> sourceRawData = Map.of();
boolean result = ConfigUtils.processSource(includeDefaultProfileData, environment, sourceName, sourceRawData);
Assertions.assertTrue(result);
}
/**
* this case is not very "interesting" because 'includeDefaultProfileData = true'
* which denotes a request not from config server; and such cases are tested in
* various other tests, before we fixed:
* https://github.com/spring-cloud/spring-cloud-kubernetes/pull/1600
*/
@Test
void testProcessNamedDataOne() {
List<StrippedSourceContainer> strippedSources = List
.of(new StrippedSourceContainer(Map.of(), "configmap-a", Map.of("one", "1")));
Environment environment = new MockEnvironment();
LinkedHashSet<String> sourceNames = new LinkedHashSet<>(List.of("configmap-a"));
String namespace = "namespace-a";
boolean decode = false;
boolean includeDefaultProfileData = true;
MultipleSourcesContainer result = ConfigUtils.processNamedData(strippedSources, environment, sourceNames,
namespace, decode, includeDefaultProfileData);
Assertions.assertNotNull(result);
Assertions.assertEquals(result.names().toString(), "[configmap-a]");
Assertions.assertEquals(result.data(), Map.of("one", "1"));
}
/**
* <pre>
* - includeDefaultProfileData = false
* - emptyActiveProfiles = false
* - profileBasedSourceName = false
* - defaultProfilePresent = true
* - rawDataContainsProfileBasedSource = does not matter
* </pre>
*
* Since 'defaultProfilePresent=true', this method must return 'true'.
*/
@Test
void testProcessSourceTwo() {
boolean includeDefaultProfileData = false;
MockEnvironment environment = new MockEnvironment();
environment.setActiveProfiles("default");
String sourceName = "account";
Map<String, String> sourceRawData = Map.of();
boolean result = ConfigUtils.processSource(includeDefaultProfileData, environment, sourceName, sourceRawData);
Assertions.assertTrue(result);
}
/**
* <pre>
* - request is coming from config server
* - activeProfile = ['default']
* - sourceName = 'account'
* </pre>
*
* As such the above will generate:
*
* <pre>
* - includeDefaultProfileData = false
* - emptyActiveProfiles = false
* - profileBasedSourceName = false
* - defaultProfilePresent = true
* - rawDataContainsProfileBasedSource = does not matter
* </pre>
*
* In this case, three types of properties will be read from the source:
*
* <pre>
* - all simple properties
* - all nested ones (yaml/yml/properties themselves) that match "${SOURCE_NAME}.{EXTENSION}"
* (in our case 'account.properties')
* - all nested ones (yaml/yml/properties themselves) that match "${SOURCE_NAME}-${ACTIVE_PROFILE}.{EXTENSION}"
* (in our case 'account-default.properties')
* - there are strict sorting rules if both of the above are matched
* </pre>
*/
@Test
void testProcessNamedDataTwo(CapturedOutput output) {
// @formatter:off
Map<String, String> sourceRawData = Map.of(
"one", "1",
"two", "2",
"account-default.properties", "five=5",
"account.properties", "one=11\nthree=3",
"account-k8s.properties", "one=22\nfour=4"
);
// @formatter:on
String sourceName = "account";
List<StrippedSourceContainer> strippedSources = List
.of(new StrippedSourceContainer(Map.of(), sourceName, sourceRawData));
MockEnvironment environment = new MockEnvironment().withProperty("spring.application.name", sourceName);
environment.setActiveProfiles("default");
LinkedHashSet<String> sourceNames = new LinkedHashSet<>(List.of(sourceName));
String namespace = "namespace-a";
boolean decode = false;
boolean includeDefaultProfileData = false;
MultipleSourcesContainer result = ConfigUtils.processNamedData(strippedSources, environment, sourceNames,
namespace, decode, includeDefaultProfileData);
Assertions.assertNotNull(result);
Assertions.assertEquals(result.names().toString(), "[account]");
/**
* <pre>
* there are some things to see here:
*
* 1. 'three=3' is present in the result, which means we have read 'account.properties'
* 2. 'five=5' is present in the result, which means we have read 'account-default.properties'
* 3. even if we have read 'account.properties', we have 'one=1' (and not 'one=11'),
* since simple properties override the ones from yml/yaml/properties
* 4. 'four=4' is not present in the result, because we do not read 'account-k8s.properties',
* since 'k8s' is not an active profile.
* </pre>
*/
Assertions.assertEquals(result.data(), Map.of("one", "1", "two", "2", "three", "3", "five", "5"));
Assertions.assertTrue(output.getOut().contains("entry : account-k8s.properties will be skipped"));
}
/**
* <pre>
* - includeDefaultProfileData = false
* - emptyActiveProfiles = false
* - profileBasedSourceName = true
* - defaultProfilePresent = false
* - rawDataContainsProfileBasedSource = does not matter
* </pre>
*
* Since 'profileBasedSourceName=true', this method must return 'true'.
*/
@Test
void testProcessSourceThree() {
boolean includeDefaultProfileData = false;
MockEnvironment environment = new MockEnvironment();
environment.setActiveProfiles("default");
String sourceName = "account-default";
Map<String, String> sourceRawData = Map.of();
boolean result = ConfigUtils.processSource(includeDefaultProfileData, environment, sourceName, sourceRawData);
Assertions.assertTrue(result);
}
/**
* <pre>
* - request is coming from config server
* - activeProfile = ['default']
* - sourceName = 'account-default'
* </pre>
*
* As such the above will generate:
*
* <pre>
* - includeDefaultProfileData = false
* - emptyActiveProfiles = false
* - profileBasedSourceName = true
* - defaultProfilePresent = does not matter
* - rawDataContainsProfileBasedSource = does not matter
* </pre>
*
* In this case, three types of properties will be read from the source:
*
* <pre>
* - all simple properties
* - all nested ones (yaml/yml/properties themselves) that match "${SOURCE_NAME}.{EXTENSION}"
* (in our case 'account.properties')
* - all nested ones (yaml/yml/properties themselves) that match "${SOURCE_NAME}-${ACTIVE_PROFILE}.{EXTENSION}"
* (in our case 'account-default.properties')
* - there are strict sorting rules if both of the above are matched
* </pre>
*/
@Test
void testProcessNamedDataThree(CapturedOutput output) {
// @formatter:off
Map<String, String> sourceRawData = Map.of(
"one", "1",
"two", "2",
"account.properties", "one=11\nthree=3",
"account-default.properties", "one=111\nfive=5",
"account-k8s.properties", "one=22\nfour=4"
);
// @formatter:on
String sourceName = "account-default";
List<StrippedSourceContainer> strippedSources = List
.of(new StrippedSourceContainer(Map.of(), sourceName, sourceRawData));
MockEnvironment environment = new MockEnvironment().withProperty("spring.application.name", "account");
environment.setActiveProfiles("default");
LinkedHashSet<String> sourceNames = new LinkedHashSet<>(List.of(sourceName));
String namespace = "namespace-a";
boolean decode = false;
boolean includeDefaultProfileData = false;
MultipleSourcesContainer result = ConfigUtils.processNamedData(strippedSources, environment, sourceNames,
namespace, decode, includeDefaultProfileData);
Assertions.assertNotNull(result);
Assertions.assertEquals(result.names().toString(), "[account-default]");
/**
* <pre>
* there are some things to see here:
*
* 1. 'one=1' is present in the result, which means we have read simple properties.
* 2. 'two-2' is present in the result, which means we have read simple properties.
* 3. even if we have read 'account.properties', we have 'one=1' (and not 'one=11'),
* since simple properties override the ones from yml/yaml/properties
* 4. 'three=3' is present in the result, which means we have read 'account.properties'.
* 5. 'five=5' is present in the result, which means we have read 'account-default.properties'
* (but we don't have 'one=111', instead : 'one=1')
* 6. we do not have 'four=4' since we do not read 'account-k8s.properties'
* </pre>
*/
Assertions.assertEquals(result.data(), Map.of("one", "1", "two", "2", "three", "3", "five", "5"));
Assertions.assertTrue(output.getOut().contains("entry : account-k8s.properties will be skipped"));
}
/**
* <pre>
* - includeDefaultProfileData = false
* - emptyActiveProfiles = false
* - profileBasedSourceName = false
* - defaultProfilePresent = false
* - rawDataContainsProfileBasedSource = false
* </pre>
*
*/
@Test
void testProcessSourceFour() {
boolean includeDefaultProfileData = false;
MockEnvironment environment = new MockEnvironment();
environment.setActiveProfiles("k8s");
String sourceName = "account";
Map<String, String> sourceRawData = Map.of("one", "1");
boolean result = ConfigUtils.processSource(includeDefaultProfileData, environment, sourceName, sourceRawData);
Assertions.assertFalse(result);
}
/**
* <pre>
* - includeDefaultProfileData = false
* - emptyActiveProfiles = false
* - profileBasedSourceName = false
* - defaultProfilePresent = false
* - rawDataContainsProfileBasedSource = true
* </pre>
*
*/
@Test
void testProcessSourceFive() {
boolean includeDefaultProfileData = false;
MockEnvironment environment = new MockEnvironment();
environment.setActiveProfiles("k8s");
String sourceName = "account";
Map<String, String> sourceRawData = Map.of("one", "1", "account-k8s.properties", "one=11");
boolean result = ConfigUtils.processSource(includeDefaultProfileData, environment, sourceName, sourceRawData);
Assertions.assertTrue(result);
}
/**
* <pre>
* - request is coming from config server
* - activeProfile = ['k8s']
* - sourceName = 'account'
* - rawData inside the source has an entry : 'account-k8s.properties'
* </pre>
*
* As such the above will generate:
*
* <pre>
* - includeDefaultProfileData = false
* - emptyActiveProfiles = false
* - profileBasedSourceName = false
* - defaultProfilePresent = false
* - rawDataContainsProfileBasedSource = true
* </pre>
*
* In this case, only one type of source data will be read
*
* <pre>
* - "${SOURCE_NAME}-${ACTIVE_PROFILE}.{EXTENSION}"
* </pre>
*/
@Test
void testProcessNamedDataFive(CapturedOutput output) {
// @formatter:off
Map<String, String> sourceRawData = Map.of(
"one", "1",
"account.properties", "one=11\ntwo=2",
"account-default.properties", "one=111\nthree=3",
"account-k8s.properties", "one=1111\nfour=4"
);
// @formatter:on
String sourceName = "account";
List<StrippedSourceContainer> strippedSources = List
.of(new StrippedSourceContainer(Map.of(), sourceName, sourceRawData));
MockEnvironment environment = new MockEnvironment().withProperty("spring.application.name", sourceName);
environment.setActiveProfiles("k8s");
LinkedHashSet<String> sourceNames = new LinkedHashSet<>(List.of(sourceName));
String namespace = "namespace-a";
boolean decode = false;
boolean includeDefaultProfileData = false;
MultipleSourcesContainer result = ConfigUtils.processNamedData(strippedSources, environment, sourceNames,
namespace, decode, includeDefaultProfileData);
Assertions.assertNotNull(result);
Assertions.assertEquals(result.names().toString(), "[account]");
/**
* <pre>
* - we only read from 'account-k8s.properties'
* </pre>
*/
Assertions.assertEquals(result.data(), Map.of("one", "1111", "four", "4"));
Assertions.assertTrue(output.getOut().contains("entry : account.properties will be skipped"));
Assertions.assertTrue(output.getOut().contains("entry : account-default.properties will be skipped"));
}
/**
* <pre>
* - includeDefaultProfileData = false
* - emptyActiveProfiles = false
* - profileBasedSourceName = true
* - defaultProfilePresent = does not matter
* - rawDataContainsProfileBasedSource = does not matter
* </pre>
*
*/
@Test
void testProcessSourceSix() {
boolean includeDefaultProfileData = false;
MockEnvironment environment = new MockEnvironment();
environment.setActiveProfiles("k8s");
String sourceName = "account-k8s";
Map<String, String> sourceRawData = Map.of("one", "1", "account-k8s.properties", "one=11");
boolean result = ConfigUtils.processSource(includeDefaultProfileData, environment, sourceName, sourceRawData);
Assertions.assertTrue(result);
}
/**
* <pre>
* - request is coming from config server
* - activeProfile = ['k8s']
* - sourceName = 'account-k8s'
* </pre>
*
* As such the above will generate:
*
* <pre>
* - includeDefaultProfileData = false
* - emptyActiveProfiles = false
* - profileBasedSourceName = true
* - defaultProfilePresent = does not matter
* - rawDataContainsProfileBasedSource = does not matter
* </pre>
*
* In this case, a few types of data will be read:
*
* <pre>
* - all simple properties
* - all nested ones (yaml/yml/properties themselves) that match "${SOURCE_NAME}.{EXTENSION}"
* (in our case 'account.properties')
* - all nested ones (yaml/yml/properties themselves) that match "${SOURCE_NAME}-${ACTIVE_PROFILE}.{EXTENSION}"
* (in our case 'account-k8s.properties')
* - there are strict sorting rules if both of the above are matched
* </pre>
*/
@Test
void testProcessNamedDataSix(CapturedOutput output) {
// @formatter:off
Map<String, String> sourceRawData = Map.of(
"one", "1",
"two", "2",
"account.properties", "one=11\nthree=3",
"account-default.properties", "one=111\nfour=4",
"account-k8s.properties", "one=1111\nfive=5",
"account-prod.properties", "six=6"
);
// @formatter:on
String sourceName = "account-k8s";
List<StrippedSourceContainer> strippedSources = List
.of(new StrippedSourceContainer(Map.of(), sourceName, sourceRawData));
MockEnvironment environment = new MockEnvironment().withProperty("spring.application.name", "account");
environment.setActiveProfiles("k8s");
LinkedHashSet<String> sourceNames = new LinkedHashSet<>(List.of(sourceName));
String namespace = "namespace-a";
boolean decode = false;
boolean includeDefaultProfileData = false;
MultipleSourcesContainer result = ConfigUtils.processNamedData(strippedSources, environment, sourceNames,
namespace, decode, includeDefaultProfileData);
Assertions.assertNotNull(result);
Assertions.assertEquals(result.names().toString(), "[account-k8s]");
/**
* <pre>
* there are some things to see here:
*
* 1. 'one=1' is not present in the result, we are not supposed to read simple properties.
* 2. 'two-2' is not present in the result, we are not supposed to read simple properties.
* 3. even if we have read 'account.properties', we have 'one=1' (and not 'one=11'),
* since simple properties override the ones from yml/yaml/properties
* 4. 'three=3' is not present in the result, we are not supposed to read '${SPRING>APPLICATION.NAME}'
* properties.
* 5. 'four=4' is not present in the result, which means we have not read 'account-default.properties'
* (but we don't have 'one=111', instead : 'one=1')
* 6. we do not have 'three=3' since we do not read 'account-default.properties'
* 7. we do not have 'six=6' since we do not read 'account-prod.properties'
* (because 'prod' is not an active profile)
* </pre>
*/
Assertions.assertEquals(result.data(), Map.of("one", "1111", "five", "5"));
Assertions.assertTrue(output.getOut().contains("entry : account-prod.properties will be skipped"));
Assertions.assertTrue(output.getOut().contains("entry : account.properties will be skipped"));
Assertions.assertTrue(output.getOut().contains("entry : account-default.properties will be skipped"));
}
}

View File

@@ -0,0 +1,95 @@
/*
* Copyright 2013-2024 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.cloud.kubernetes.commons.config;
import java.util.List;
import java.util.Map;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
/**
* Class that is supposed to test only ConfigUtils::rawDataContainsProfileBasedSource
*
* @author wind57
*/
class ConfigUtilsRawDataContainsProfileBasedSourceTests {
@Test
void nullSourceRawData() {
List<String> activeProfiles = List.of();
Map<String, String> rawData = null;
boolean result = ConfigUtils.rawDataContainsProfileBasedSource(activeProfiles, rawData).getAsBoolean();
Assertions.assertFalse(result);
}
@Test
void sourceRawDataPresentEmptyActiveProfiles() {
List<String> activeProfiles = List.of();
Map<String, String> rawData = Map.of("account-k8s.yaml", "value");
boolean result = ConfigUtils.rawDataContainsProfileBasedSource(activeProfiles, rawData).getAsBoolean();
Assertions.assertFalse(result);
}
@Test
void plainValuesOnly() {
List<String> activeProfiles = List.of("k8s");
Map<String, String> rawData = Map.of("account", "value");
boolean result = ConfigUtils.rawDataContainsProfileBasedSource(activeProfiles, rawData).getAsBoolean();
Assertions.assertFalse(result);
}
@Test
void noMatchInActiveProfiles() {
List<String> activeProfiles = List.of("k8s");
Map<String, String> rawData = Map.of("account-dev.yml", "value");
boolean result = ConfigUtils.rawDataContainsProfileBasedSource(activeProfiles, rawData).getAsBoolean();
Assertions.assertFalse(result);
}
@Test
void matchInActiveProfilesWithYml() {
List<String> activeProfiles = List.of("dev");
Map<String, String> rawData = Map.of("account-dev.yml", "value");
boolean result = ConfigUtils.rawDataContainsProfileBasedSource(activeProfiles, rawData).getAsBoolean();
Assertions.assertTrue(result);
}
@Test
void matchInActiveProfilesWithYaml() {
List<String> activeProfiles = List.of("dev", "k8s");
Map<String, String> rawData = Map.of("account-dev.yaml", "value");
boolean result = ConfigUtils.rawDataContainsProfileBasedSource(activeProfiles, rawData).getAsBoolean();
Assertions.assertTrue(result);
}
@Test
void matchInActiveProfilesWithProperties() {
List<String> activeProfiles = List.of("dev", "k8s");
Map<String, String> rawData = Map.of("account-dev.properties", "value");
boolean result = ConfigUtils.rawDataContainsProfileBasedSource(activeProfiles, rawData).getAsBoolean();
Assertions.assertTrue(result);
}
}