Server-side resolve Vault secrets inside application's property sources (#1678)

* Added ability to access vault secrets from application config properties.

Co-authored-by: Spencer Gibb <sgibb@pivotal.io>

spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/config/ConfigServerMvcConfiguration.java

@@ -17,6 +17,7 @@
 package org.springframework.cloud.config.server.config;
 
 import java.util.HashMap;
+import java.util.List;
 import java.util.Map;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
@@ -60,7 +61,7 @@ public class ConfigServerMvcConfiguration implements WebMvcConfigurer {
 	static class EnvironmentControllerConfiguration {
 
 		@Autowired(required = false)
-		private EnvironmentEncryptor environmentEncryptor;
+		private List<EnvironmentEncryptor> environmentEncryptors;
 
 		@Autowired(required = false)
 		private Map<String, ResourceEncryptor> resourceEncryptorMap = new HashMap<>();
@@ -96,7 +97,7 @@ public class ConfigServerMvcConfiguration implements WebMvcConfigurer {
 
 		private EnvironmentRepository encrypted(EnvironmentRepository envRepository, ConfigServerProperties server) {
 			EnvironmentEncryptorEnvironmentRepository encrypted = new EnvironmentEncryptorEnvironmentRepository(
-					envRepository, this.environmentEncryptor);
+					envRepository, this.environmentEncryptors);
 			encrypted.setOverrides(server.getOverrides());
 			return encrypted;
 		}

spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/config/VaultEncryptionAutoConfiguration.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright 2020-2020 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.cloud.config.server.config;
+
+import org.springframework.boot.autoconfigure.condition.ConditionalOnClass;
+import org.springframework.cloud.config.server.encryption.vault.VaultEnvironmentEncryptor;
+import org.springframework.cloud.config.server.environment.vault.SpringVaultEnvironmentRepository;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.context.annotation.Profile;
+import org.springframework.vault.core.VaultTemplate;
+
+/**
+ * Auto configuration for vault encryptor.
+ *
+ * @author Alexey Zhokhov
+ */
+@Configuration(proxyBeanMethods = false)
+@ConditionalOnClass(VaultTemplate.class)
+@Profile("vault")
+public class VaultEncryptionAutoConfiguration {
+
+	@Bean
+	public VaultEnvironmentEncryptor vaultEnvironmentEncryptor(
+			SpringVaultEnvironmentRepository vaultEnvironmentRepository) {
+		return new VaultEnvironmentEncryptor(
+				vaultEnvironmentRepository.getKeyValueTemplate());
+	}
+
+}

spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/encryption/vault/VaultEnvironmentEncryptor.java

@@ -0,0 +1,128 @@
+/*
+ * Copyright 2018-2019 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.cloud.config.server.encryption.vault;
+
+import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.Map;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import org.springframework.cloud.config.environment.Environment;
+import org.springframework.cloud.config.environment.PropertySource;
+import org.springframework.cloud.config.server.encryption.CipherEnvironmentEncryptor;
+import org.springframework.cloud.config.server.encryption.EnvironmentEncryptor;
+import org.springframework.util.StringUtils;
+import org.springframework.vault.core.VaultKeyValueOperations;
+import org.springframework.vault.support.VaultResponse;
+
+/**
+ * VaultEnvironmentEncryptor that can decrypt property values prefixed with {vault}
+ * marker.
+ *
+ * @author Alexey Zhokhov
+ */
+public class VaultEnvironmentEncryptor implements EnvironmentEncryptor {
+
+	private static final Log logger = LogFactory.getLog(CipherEnvironmentEncryptor.class);
+
+	private final VaultKeyValueOperations keyValueTemplate;
+
+	public VaultEnvironmentEncryptor(VaultKeyValueOperations keyValueTemplate) {
+		this.keyValueTemplate = keyValueTemplate;
+	}
+
+	@Override
+	public Environment decrypt(Environment environment) {
+		Map<String, VaultResponse> loadedVaultKeys = new HashMap<>();
+
+		Environment result = new Environment(environment);
+		for (PropertySource source : environment.getPropertySources()) {
+			Map<Object, Object> map = new LinkedHashMap<>(source.getSource());
+			for (Map.Entry<Object, Object> entry : new LinkedHashSet<>(map.entrySet())) {
+				Object key = entry.getKey();
+				String name = key.toString();
+				if (entry.getValue() != null
+						&& entry.getValue().toString().startsWith("{vault}")) {
+					String value = entry.getValue().toString();
+					map.remove(key);
+					try {
+						value = value.substring("{vault}".length());
+
+						if (!value.startsWith(":")) {
+							throw new RuntimeException("Wrong format");
+						}
+
+						value = value.substring(1);
+
+						if (!value.contains("#")) {
+							throw new RuntimeException("Wrong format");
+						}
+
+						String[] parts = value.split("#");
+
+						if (parts.length == 1) {
+							throw new RuntimeException("Wrong format");
+						}
+
+						if (StringUtils.isEmpty(parts[0])
+								|| StringUtils.isEmpty(parts[1])) {
+							throw new RuntimeException("Wrong format");
+						}
+
+						String vaultKey = parts[0];
+						String vaultParamName = parts[1];
+
+						if (!loadedVaultKeys.containsKey(vaultKey)) {
+							loadedVaultKeys.put(vaultKey, keyValueTemplate.get(vaultKey));
+						}
+
+						VaultResponse vaultResponse = loadedVaultKeys.get(vaultKey);
+
+						if (vaultResponse == null
+								|| (vaultResponse.getData() == null || !vaultResponse
+										.getData().containsKey(vaultParamName))) {
+							value = null;
+						}
+						else {
+							value = vaultResponse.getData().get(vaultParamName)
+									.toString();
+						}
+					}
+					catch (Exception e) {
+						value = "<n/a>";
+						name = "invalid." + name;
+						String message = "Cannot resolve key: " + key + " ("
+								+ e.getClass() + ": " + e.getMessage() + ")";
+						if (logger.isDebugEnabled()) {
+							logger.debug(message, e);
+						}
+						else if (logger.isWarnEnabled()) {
+							logger.warn(message);
+						}
+					}
+					map.put(name, value);
+				}
+			}
+			result.add(new PropertySource(source.getName(), map));
+		}
+		return result;
+	}
+
+}

spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/EnvironmentEncryptorEnvironmentRepository.java

@@ -18,6 +18,7 @@ package org.springframework.cloud.config.server.environment;
 
 import java.util.HashMap;
 import java.util.LinkedHashMap;
+import java.util.List;
 import java.util.Map;
 
 import org.springframework.cloud.config.environment.Environment;
@@ -36,7 +37,7 @@ public class EnvironmentEncryptorEnvironmentRepository implements EnvironmentRep
 
 	private EnvironmentRepository delegate;
 
-	private EnvironmentEncryptor environmentEncryptor;
+	private final List<EnvironmentEncryptor> environmentEncryptors;
 
 	private Map<String, String> overrides = new LinkedHashMap<>();
 
@@ -45,9 +46,9 @@ public class EnvironmentEncryptorEnvironmentRepository implements EnvironmentRep
 	}
 
 	public EnvironmentEncryptorEnvironmentRepository(EnvironmentRepository delegate,
-			EnvironmentEncryptor environmentEncryptor) {
+			List<EnvironmentEncryptor> environmentEncryptors) {
 		this.delegate = delegate;
-		this.environmentEncryptor = environmentEncryptor;
+		this.environmentEncryptors = environmentEncryptors;
 	}
 
 	@Override
@@ -58,8 +59,10 @@ public class EnvironmentEncryptorEnvironmentRepository implements EnvironmentRep
 	@Override
 	public Environment findOne(String name, String profiles, String label, boolean includeOrigin) {
 		Environment environment = this.delegate.findOne(name, profiles, label, includeOrigin);
-		if (this.environmentEncryptor != null) {
-			environment = this.environmentEncryptor.decrypt(environment);
+		if (this.environmentEncryptors != null) {
+			for (EnvironmentEncryptor environmentEncryptor : environmentEncryptors) {
+				environment = environmentEncryptor.decrypt(environment);
+			}
 		}
 		if (!this.overrides.isEmpty()) {
 			environment.addFirst(new PropertySource("overrides", getOverridesMap(includeOrigin)));

spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/vault/SpringVaultEnvironmentRepository.java

@@ -59,7 +59,7 @@ public class SpringVaultEnvironmentRepository extends AbstractVaultEnvironmentRe
 		return null;
 	}
 
-	VaultKeyValueOperations getKeyValueTemplate() {
+	public VaultKeyValueOperations getKeyValueTemplate() {
 		return this.keyValueTemplate;
 	}
 

spring-cloud-config-server/src/main/resources/META-INF/spring.factories

@@ -8,6 +8,7 @@ org.springframework.cloud.config.server.bootstrap.ConfigServerBootstrapApplicati
 # Autoconfiguration
 org.springframework.boot.autoconfigure.EnableAutoConfiguration=\
 org.springframework.cloud.config.server.config.ConfigServerAutoConfiguration,\
-org.springframework.cloud.config.server.config.EncryptionAutoConfiguration
+org.springframework.cloud.config.server.config.EncryptionAutoConfiguration,\
+org.springframework.cloud.config.server.config.VaultEncryptionAutoConfiguration
 org.springframework.boot.diagnostics.FailureAnalyzer=\
 org.springframework.cloud.config.server.diagnostics.GitUriFailureAnalyzer

spring-cloud-config-server/src/test/java/org/springframework/cloud/config/server/encryption/vault/VaultEnvironmentEncryptorTests.java

@@ -0,0 +1,209 @@
+/*
+ * Copyright 2020-2020 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.cloud.config.server.encryption.vault;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.junit.Test;
+
+import org.springframework.cloud.config.environment.Environment;
+import org.springframework.cloud.config.environment.PropertySource;
+import org.springframework.vault.core.VaultKeyValueOperations;
+import org.springframework.vault.support.VaultResponse;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * @author Alexey Zhokhov
+ */
+public class VaultEnvironmentEncryptorTests {
+
+	@Test
+	public void shouldResolveProperty() {
+		// given
+		String secret = "mysecret";
+
+		VaultKeyValueOperations keyValueTemplate = mock(VaultKeyValueOperations.class);
+		when(keyValueTemplate.get("accounts/mypay"))
+				.thenReturn(withVaultResponse("access_key", secret));
+
+		VaultEnvironmentEncryptor encryptor = new VaultEnvironmentEncryptor(
+				keyValueTemplate);
+
+		// when
+		Environment environment = new Environment("name", "profile", "label");
+		environment.add(new PropertySource("a", Collections.<Object, Object>singletonMap(
+				environment.getName(), "{vault}:accounts/mypay#access_key")));
+
+		// then
+		assertThat(encryptor.decrypt(environment).getPropertySources().get(0).getSource()
+				.get(environment.getName())).isEqualTo(secret);
+	}
+
+	@Test
+	public void shouldReturnNullIfPropertyNotFoundInVault() {
+		// given
+		String secret = "mysecret";
+
+		VaultKeyValueOperations keyValueTemplate = mock(VaultKeyValueOperations.class);
+		when(keyValueTemplate.get("accounts/mypay"))
+				.thenReturn(withVaultResponse("access_key", secret));
+
+		VaultEnvironmentEncryptor encryptor = new VaultEnvironmentEncryptor(
+				keyValueTemplate);
+
+		// when
+		Environment environment = new Environment("name", "profile", "label");
+		environment.add(new PropertySource("a", Collections.<Object, Object>singletonMap(
+				environment.getName(), "{vault}:accounts/mypay#another_key")));
+
+		// then
+		assertThat(encryptor.decrypt(environment).getPropertySources().get(0).getSource()
+				.get(environment.getName())).isNull();
+	}
+
+	@Test
+	public void shouldSkipPropertyWithNotVaultPrefix() {
+		// given
+		String value = "test{vault}:accounts/mypay#access_key";
+
+		VaultKeyValueOperations keyValueTemplate = mock(VaultKeyValueOperations.class);
+
+		VaultEnvironmentEncryptor encryptor = new VaultEnvironmentEncryptor(
+				keyValueTemplate);
+
+		// when
+		Environment environment = new Environment("name", "profile", "label");
+		environment.add(new PropertySource("a",
+				Collections.<Object, Object>singletonMap(environment.getName(), value)));
+
+		// then
+		assertThat(encryptor.decrypt(environment).getPropertySources().get(0).getSource()
+				.get(environment.getName())).isEqualTo(value);
+	}
+
+	@Test
+	public void shouldMarkAsInvalidPropertyWithNoKeyValue() {
+		// given
+		String value = "{vault}:accounts/mypay";
+
+		VaultKeyValueOperations keyValueTemplate = mock(VaultKeyValueOperations.class);
+
+		VaultEnvironmentEncryptor encryptor = new VaultEnvironmentEncryptor(
+				keyValueTemplate);
+
+		// when
+		Environment environment = new Environment("name", "profile", "label");
+		environment.add(new PropertySource("a",
+				Collections.<Object, Object>singletonMap(environment.getName(), value)));
+
+		// then
+		Environment processedEnvironment = encryptor.decrypt(environment);
+
+		assertThat(processedEnvironment.getPropertySources().get(0).getSource()
+				.get(environment.getName())).isNull();
+		assertThat(processedEnvironment.getPropertySources().get(0).getSource()
+				.get("invalid." + environment.getName())).isEqualTo("<n/a>");
+	}
+
+	@Test
+	public void shouldMarkAsInvalidPropertyWithNoEmptyValue() {
+		// given
+		String value = "{vault}:accounts/mypay#";
+
+		VaultKeyValueOperations keyValueTemplate = mock(VaultKeyValueOperations.class);
+
+		VaultEnvironmentEncryptor encryptor = new VaultEnvironmentEncryptor(
+				keyValueTemplate);
+
+		// when
+		Environment environment = new Environment("name", "profile", "label");
+		environment.add(new PropertySource("a",
+				Collections.<Object, Object>singletonMap(environment.getName(), value)));
+
+		// then
+		Environment processedEnvironment = encryptor.decrypt(environment);
+
+		assertThat(processedEnvironment.getPropertySources().get(0).getSource()
+				.get(environment.getName())).isNull();
+		assertThat(processedEnvironment.getPropertySources().get(0).getSource()
+				.get("invalid." + environment.getName())).isEqualTo("<n/a>");
+	}
+
+	@Test
+	public void shouldMarkAsInvalidPropertyWithWrongFormat() {
+		// given
+		String value = "{vault}test:accounts/mypay#";
+
+		VaultKeyValueOperations keyValueTemplate = mock(VaultKeyValueOperations.class);
+
+		VaultEnvironmentEncryptor encryptor = new VaultEnvironmentEncryptor(
+				keyValueTemplate);
+
+		// when
+		Environment environment = new Environment("name", "profile", "label");
+		environment.add(new PropertySource("a",
+				Collections.<Object, Object>singletonMap(environment.getName(), value)));
+
+		// then
+		Environment processedEnvironment = encryptor.decrypt(environment);
+
+		assertThat(processedEnvironment.getPropertySources().get(0).getSource()
+				.get(environment.getName())).isNull();
+		assertThat(processedEnvironment.getPropertySources().get(0).getSource()
+				.get("invalid." + environment.getName())).isEqualTo("<n/a>");
+	}
+
+	@Test
+	public void shouldMarkAsInvalidPropertyWithWrongFormat2() {
+		// given
+		String value = "{vault}:#xxx";
+
+		VaultKeyValueOperations keyValueTemplate = mock(VaultKeyValueOperations.class);
+
+		VaultEnvironmentEncryptor encryptor = new VaultEnvironmentEncryptor(
+				keyValueTemplate);
+
+		// when
+		Environment environment = new Environment("name", "profile", "label");
+		environment.add(new PropertySource("a",
+				Collections.<Object, Object>singletonMap(environment.getName(), value)));
+
+		// then
+		Environment processedEnvironment = encryptor.decrypt(environment);
+
+		assertThat(processedEnvironment.getPropertySources().get(0).getSource()
+				.get(environment.getName())).isNull();
+		assertThat(processedEnvironment.getPropertySources().get(0).getSource()
+				.get("invalid." + environment.getName())).isEqualTo("<n/a>");
+	}
+
+	private VaultResponse withVaultResponse(String key, Object value) {
+		Map<String, Object> responseData = new HashMap<>();
+		responseData.put(key, value);
+
+		VaultResponse response = new VaultResponse();
+		response.setData(responseData);
+
+		return response;
+	}
+
+}