Add simple implementation of JDBC environment repository

docs/src/main/asciidoc/spring-cloud-config.adoc

@@ -571,7 +571,7 @@ while providing tight access control and recording a detailed audit log.
 
 For more information on Vault see the https://www.vaultproject.io/intro/index.html[Vault quickstart guide].
 
-To enable the config server to use a Vault backend you must run your config server
+To enable the config server to use a Vault backend you can run your config server
 with the `vault` profile.  For example in your config server's `application.properties`
 you can add `spring.profiles.active=vault`.
 
@@ -710,10 +710,30 @@ $ vault write secret/application foo=bar baz=bam
 All applications using the config server will have the properties
 `foo` and `baz` available to them.
 
+==== JDBC Backend
+
+Spring Cloud Config Server supports JDBC (relation database) as a
+backend for configuration properties. You can enable this feature by
+adding `spring-jdbc` to the classpath, and using the "jdbc" profile,
+or by adding a bean of type `JdbcEnvironmentRepository`. Spring Boot
+will configure a data source if you include the right dependencies on
+the classpath (see the user guide for more details on that).
+
+The database needs to have a table called "PROPERTIES" with columns
+"APPLICATION", "PROFILE", "LABEL" (with the usual `Environment`
+meaning), plus "KEY" and "VALUE" for the key and value pairs in
+`Properties` style. All fields are of type String in Java, so you can
+make them `VARCHAR` of whatever length you need. Property values
+behave in the same way as they would if they came from Spring Boot
+properties files named `{application}-{profile}.properties`, including
+all the encryption and decryption, which will be applied as
+post-processing steps (i.e. not in the repository implementation
+directly).
+
 ==== Composite Environment Repositories
 
 In some scenarios you may wish to pull configuration data from multiple
-environment repositories.  To do this just enable
+environment repositories.  To do this you can just enable
 multiple profiles in your config server's application properties or YAML file.
 If, for example, you want to pull configuration data from a Git repository
 as well as a SVN repository you would set the following properties for your

pom.xml

@@ -11,7 +11,7 @@
 	<parent>
 		<groupId>org.springframework.cloud</groupId>
 		<artifactId>spring-cloud-build</artifactId>
-		<version>1.3.5.BUILD-SNAPSHOT</version>
+		<version>1.3.5.RELEASE</version>
 		<relativePath/> <!-- lookup parent from repository -->
 	</parent>
 	<scm>

spring-cloud-config-server/pom.xml

@@ -22,6 +22,11 @@
 			<artifactId>spring-boot-configuration-processor</artifactId>
 			<optional>true</optional>
 		</dependency>
+		<dependency>
+			<groupId>org.springframework.boot</groupId>
+			<artifactId>spring-boot-starter-jdbc</artifactId>
+			<optional>true</optional>
+		</dependency>
 		<dependency>
 			<groupId>org.springframework.cloud</groupId>
 			<artifactId>spring-cloud-config-client</artifactId>
@@ -60,6 +65,11 @@
 			<artifactId>aws-java-sdk-core</artifactId>
 			<optional>true</optional>
 		</dependency>
+		<dependency>
+			<groupId>com.h2database</groupId>
+			<artifactId>h2</artifactId>
+			<scope>test</scope>
+		</dependency>
 		<dependency>
 			<groupId>org.springframework.boot</groupId>
 			<artifactId>spring-boot-starter-test</artifactId>

spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/config/EnvironmentRepositoryConfiguration.java

@@ -18,12 +18,14 @@ package org.springframework.cloud.config.server.config;
 import javax.servlet.http.HttpServletRequest;
 
 import org.eclipse.jgit.api.TransportConfigCallback;
+
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
 import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
 import org.springframework.cloud.config.server.environment.ConsulEnvironmentWatch;
 import org.springframework.cloud.config.server.environment.EnvironmentRepository;
 import org.springframework.cloud.config.server.environment.EnvironmentWatch;
+import org.springframework.cloud.config.server.environment.JdbcEnvironmentRepository;
 import org.springframework.cloud.config.server.environment.MultipleJGitEnvironmentRepository;
 import org.springframework.cloud.config.server.environment.NativeEnvironmentRepository;
 import org.springframework.cloud.config.server.environment.SvnKitEnvironmentRepository;
@@ -33,6 +35,7 @@ import org.springframework.context.annotation.Configuration;
 import org.springframework.context.annotation.Import;
 import org.springframework.context.annotation.Profile;
 import org.springframework.core.env.ConfigurableEnvironment;
+import org.springframework.jdbc.core.JdbcTemplate;
 import org.springframework.web.client.RestTemplate;
 
 /**
@@ -41,7 +44,7 @@ import org.springframework.web.client.RestTemplate;
  *
  */
 @Configuration
-@Import({ VaultRepositoryConfiguration.class, SvnRepositoryConfiguration.class,
+@Import({ JdbcRepositoryConfiguration.class, VaultRepositoryConfiguration.class, SvnRepositoryConfiguration.class,
 		NativeRepositoryConfiguration.class, GitRepositoryConfiguration.class,
 		DefaultRepositoryConfiguration.class })
 public class EnvironmentRepositoryConfiguration {
@@ -147,3 +150,12 @@ class VaultRepositoryConfiguration {
 		return new VaultEnvironmentRepository(request, watch, new RestTemplate());
 	}
 }
+
+@Configuration
+@Profile("jdbc")
+class JdbcRepositoryConfiguration {
+	@Bean
+	public JdbcEnvironmentRepository jdbcEnvironmentRepository(JdbcTemplate jdbc) {
+		return new JdbcEnvironmentRepository(jdbc);
+	}
+}

spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/JdbcEnvironmentRepository.java

@@ -0,0 +1,123 @@
+/*
+ * Copyright 2016-2017 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.cloud.config.server.environment;
+
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedHashMap;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Properties;
+
+import org.springframework.boot.context.properties.ConfigurationProperties;
+import org.springframework.cloud.config.environment.Environment;
+import org.springframework.cloud.config.environment.PropertySource;
+import org.springframework.core.Ordered;
+import org.springframework.dao.DataAccessException;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.jdbc.core.ResultSetExtractor;
+import org.springframework.util.StringUtils;
+
+/**
+ * An {@link EnvironmentRepository} that picks up data from a relational database. The
+ * database should have a table called "PROPERTIES" with columns "APPLICATION", "PROFILE",
+ * "LABEL" (with the usual {@link Environment} meaning), plus "KEY" and "VALUE" for the
+ * key and value pairs in {@link Properties} style. Property values behave in the same way
+ * as they would if they came from Spring Boot properties files named
+ * <code>{application}-{profile}.properties</code>, including all the encryption and
+ * decryption, which will be applied as post-processing steps (i.e. not in this repository
+ * directly).
+ * 
+ * @author Dave Syer
+ *
+ */
+@ConfigurationProperties("spring.cloud.config.server.jdbc")
+public class JdbcEnvironmentRepository implements EnvironmentRepository, Ordered {
+
+	private static final String DEFAULT_SQL = "SELECT KEY, VALUE from PROPERTIES where APPLICATION=? and PROFILE=? and LABEL=?";
+	private int order = Ordered.LOWEST_PRECEDENCE - 10;
+	private final JdbcTemplate jdbc;
+	private String sql = DEFAULT_SQL;
+	private final PropertiesResultSetExtractor extractor = new PropertiesResultSetExtractor();
+
+	public JdbcEnvironmentRepository(JdbcTemplate jdbc) {
+		this.jdbc = jdbc;
+	}
+
+	@Override
+	public Environment findOne(String application, String profile, String label) {
+		String config = application;
+		if (StringUtils.isEmpty(label)) {
+			label = "master";
+		}
+		if (StringUtils.isEmpty(profile)) {
+			profile = "default";
+		}
+		if (!profile.startsWith("default")) {
+			profile = "default," + profile;
+		}
+		String[] profiles = StringUtils.commaDelimitedListToStringArray(profile);
+		Environment environment = new Environment(application, profiles, label, null,
+				null);
+		if (!config.startsWith("application")) {
+			config = "application," + config;
+		}
+		List<String> applications = new ArrayList<String>(new LinkedHashSet<>(
+				Arrays.asList(StringUtils.commaDelimitedListToStringArray(config))));
+		List<String> envs = new ArrayList<String>(new LinkedHashSet<>(Arrays.asList(profiles)));
+		Collections.reverse(applications);
+		Collections.reverse(envs);
+		for (String app : applications) {
+			for (String env : envs) {
+				Map<String, String> next = (Map<String, String>) jdbc.query(this.sql,
+						new Object[] { app, env, label }, this.extractor);
+				if (!next.isEmpty()) {
+					environment.add(new PropertySource(app + "-" + env, next));
+				}
+			}
+		}
+		return environment;
+	}
+
+	@Override
+	public int getOrder() {
+		return order;
+	}
+
+	public void setOrder(int order) {
+		this.order = order;
+	}
+
+}
+
+class PropertiesResultSetExtractor implements ResultSetExtractor<Map<String, String>> {
+
+	@Override
+	public Map<String, String> extractData(ResultSet rs)
+			throws SQLException, DataAccessException {
+		Map<String, String> map = new LinkedHashMap<>();
+		while (rs.next()) {
+			map.put(rs.getString(1), rs.getString(2));
+		}
+		return map;
+	}
+
+}

spring-cloud-config-server/src/test/java/org/springframework/cloud/config/server/AdhocTestSuite.java

@@ -1,14 +1,44 @@
 package org.springframework.cloud.config.server;
 
-import org.junit.Ignore;
 import org.junit.runner.RunWith;
 import org.junit.runners.Suite;
 import org.junit.runners.Suite.SuiteClasses;
+
+import org.springframework.cloud.config.server.config.ConfigServerHealthIndicatorTests;
+import org.springframework.cloud.config.server.config.CustomCompositeEnvironmentRepositoryTests;
+import org.springframework.cloud.config.server.config.CustomEnvironmentRepositoryTests;
+import org.springframework.cloud.config.server.config.TransportConfigurationTest;
+import org.springframework.cloud.config.server.credentials.AwsCodeCommitCredentialsProviderTests;
+import org.springframework.cloud.config.server.credentials.GitCredentialsProviderFactoryTests;
+import org.springframework.cloud.config.server.encryption.CipherEnvironmentEncryptorTests;
+import org.springframework.cloud.config.server.encryption.EncryptionControllerMultiTextEncryptorTests;
+import org.springframework.cloud.config.server.encryption.EncryptionControllerTests;
+import org.springframework.cloud.config.server.encryption.EncryptionIntegrationTests;
+import org.springframework.cloud.config.server.encryption.EnvironmentPrefixHelperTests;
+import org.springframework.cloud.config.server.encryption.KeyStoreTextEncryptorLocatorTests;
+import org.springframework.cloud.config.server.environment.CompositeEnvironmentRepositoryTests;
+import org.springframework.cloud.config.server.environment.EnvironmentControllerIntegrationTests;
+import org.springframework.cloud.config.server.environment.EnvironmentControllerTests;
 import org.springframework.cloud.config.server.environment.EnvironmentEncryptorEnvironmentRepositoryTests;
+import org.springframework.cloud.config.server.environment.JGitEnvironmentRepositoryConcurrencyTests;
 import org.springframework.cloud.config.server.environment.JGitEnvironmentRepositoryIntegrationTests;
+import org.springframework.cloud.config.server.environment.JGitEnvironmentRepositoryTests;
+import org.springframework.cloud.config.server.environment.JdbcEnvironmentRepositoryTests;
+import org.springframework.cloud.config.server.environment.MultipleJGitEnvironmentApplicationPlaceholderRepositoryTests;
+import org.springframework.cloud.config.server.environment.MultipleJGitEnvironmentLabelPlaceholderRepositoryTests;
+import org.springframework.cloud.config.server.environment.MultipleJGitEnvironmentProfilePlaceholderRepositoryTests;
 import org.springframework.cloud.config.server.environment.MultipleJGitEnvironmentRepositoryIntegrationTests;
+import org.springframework.cloud.config.server.environment.MultipleJGitEnvironmentRepositoryTests;
 import org.springframework.cloud.config.server.environment.NativeEnvironmentRepositoryTests;
 import org.springframework.cloud.config.server.environment.SVNKitEnvironmentRepositoryIntegrationTests;
+import org.springframework.cloud.config.server.environment.SVNKitEnvironmentRepositoryTests;
+import org.springframework.cloud.config.server.environment.VaultEnvironmentRepositoryTests;
+import org.springframework.cloud.config.server.resource.GenericResourceRepositoryTests;
+import org.springframework.cloud.config.server.resource.ResourceControllerIntegrationTests;
+import org.springframework.cloud.config.server.resource.ResourceControllerTests;
+import org.springframework.cloud.config.server.ssh.PropertyBasedSshSessionFactoryTest;
+import org.springframework.cloud.config.server.ssh.SshPropertyValidatorTest;
+import org.springframework.cloud.config.server.ssh.SshUriPropertyProcessorTest;
 
 /**
  * A test suite for probing weird ordering problems in the tests.
@@ -16,10 +46,54 @@ import org.springframework.cloud.config.server.environment.SVNKitEnvironmentRepo
  * @author Dave Syer
  */
 @RunWith(Suite.class)
-@SuiteClasses({ MultipleJGitEnvironmentRepositoryIntegrationTests.class,
-		JGitEnvironmentRepositoryIntegrationTests.class, EnvironmentEncryptorEnvironmentRepositoryTests.class,
-		NativeEnvironmentRepositoryTests.class, SVNKitEnvironmentRepositoryIntegrationTests.class })
-@Ignore
+@SuiteClasses({ TransportConfigurationIntegrationTests.PropertyBasedCallbackTest.class,
+	ConfigClientOnIntegrationTests.class,
+	BootstrapConfigServerIntegrationTests.class,
+	ResourceControllerIntegrationTests.class,
+	GenericResourceRepositoryTests.class,
+	ResourceControllerTests.class,
+	SubversionConfigServerIntegrationTests.class,
+	TransportConfigurationIntegrationTests.FileBasedCallbackTest.class,
+	TransportConfigurationTest.class,
+	ConfigServerHealthIndicatorTests.class,
+	CustomCompositeEnvironmentRepositoryTests.class,
+	CustomEnvironmentRepositoryTests.class,
+	ConfigClientOffIntegrationTests.class,
+	AwsCodeCommitCredentialsProviderTests.class,
+	GitCredentialsProviderFactoryTests.class,
+	PropertyBasedSshSessionFactoryTest.class,
+	SshUriPropertyProcessorTest.class,
+	SshPropertyValidatorTest.class,
+	NativeConfigServerIntegrationTests.class,
+	EncryptionIntegrationTests.ConfigSymmetricEncryptionIntegrationTests.class,
+	EnvironmentPrefixHelperTests.class,
+	EncryptionControllerTests.class,
+	CipherEnvironmentEncryptorTests.class,
+	EncryptionIntegrationTests.KeystoreConfigurationIntegrationTests.class,
+	KeyStoreTextEncryptorLocatorTests.class,
+	EncryptionIntegrationTests.BootstrapConfigSymmetricEncryptionIntegrationTests.class,
+	EncryptionControllerMultiTextEncryptorTests.class,
+	CompositeConfigServerIntegrationTests.class,
+	VanillaConfigServerIntegrationTests.class,
+	VaultEnvironmentRepositoryTests.class,
+	MultipleJGitEnvironmentLabelPlaceholderRepositoryTests.class,
+	EnvironmentControllerTests.class,
+	JGitEnvironmentRepositoryIntegrationTests.class,
+	CompositeEnvironmentRepositoryTests.class,
+	MultipleJGitEnvironmentApplicationPlaceholderRepositoryTests.class,
+	EnvironmentControllerIntegrationTests.class,
+	JGitEnvironmentRepositoryTests.class,
+	EnvironmentEncryptorEnvironmentRepositoryTests.class,
+	JdbcEnvironmentRepositoryTests.class,
+	SVNKitEnvironmentRepositoryTests.class,
+	MultipleJGitEnvironmentRepositoryTests.class,
+	NativeEnvironmentRepositoryTests.class,
+	JGitEnvironmentRepositoryConcurrencyTests.class,
+	SVNKitEnvironmentRepositoryIntegrationTests.class,
+	MultipleJGitEnvironmentRepositoryIntegrationTests.class,
+	MultipleJGitEnvironmentProfilePlaceholderRepositoryTests.class
+ })
+// @Ignore
 public class AdhocTestSuite {
 
 }

spring-cloud-config-server/src/test/java/org/springframework/cloud/config/server/environment/EnvironmentControllerTests.java

@@ -321,7 +321,6 @@ public class EnvironmentControllerTests {
 		Mockito.when(this.repository.findOne("foo", "bar", null))
 				.thenReturn(this.environment);
 		String json = this.controller.jsonProperties("foo", "bar", false).getBody();
-		System.err.println(json);
 		assertThat("Wrong output: " + json, json, is(
 				"{\"a\":{\"b\":[{\"c\":\"x\",\"d\":[\"xx\",\"yy\"]},{\"c\":\"y\",\"e\":[{\"d\":\"z\"}]}]}}"));
 	}

spring-cloud-config-server/src/test/java/org/springframework/cloud/config/server/environment/JdbcEnvironmentRepositoryTests.java

@@ -0,0 +1,82 @@
+/*
+ * Copyright 2016-2017 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.cloud.config.server.environment;
+
+import javax.sql.DataSource;
+
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.autoconfigure.jdbc.AutoConfigureTestDatabase;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.cloud.config.environment.Environment;
+import org.springframework.cloud.config.server.environment.JdbcEnvironmentRepositoryTests.ApplicationConfiguration;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.test.annotation.DirtiesContext;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * @author Dave Syer
+ *
+ */
+@RunWith(SpringRunner.class)
+@SpringBootTest(classes = ApplicationConfiguration.class, properties = {
+		"spring.datasource.schema=classpath:schema-jdbc.sql",
+		"spring.datasource.data=classpath:data-jdbc.sql" })
+@AutoConfigureTestDatabase
+@DirtiesContext
+public class JdbcEnvironmentRepositoryTests {
+
+	@Autowired
+	private DataSource dataSource;
+
+	@Test
+	public void basicProperties() {
+		Environment env = new JdbcEnvironmentRepository(new JdbcTemplate(dataSource))
+				.findOne("foo", "bar", "");
+		assertThat(env.getName()).isEqualTo("foo");
+		assertThat(env.getProfiles()).isEqualTo(new String[] { "default", "bar" });
+		assertThat(env.getLabel()).isEqualTo("master");
+		assertThat(env.getPropertySources()).isNotEmpty();
+		assertThat(env.getPropertySources().get(0).getName()).isEqualTo("foo-bar");
+		assertThat(env.getPropertySources().get(0).getSource().get("a.b.c"))
+				.isEqualTo("x");
+		assertThat(env.getPropertySources().get(1).getName())
+				.isEqualTo("application-default");
+		assertThat(env.getPropertySources().get(1).getSource().get("a.b")).isEqualTo("y");
+	}
+
+	@Test
+	public void defaults() {
+		Environment env = new JdbcEnvironmentRepository(new JdbcTemplate(dataSource))
+				.findOne("application", "", "");
+		assertThat(env.getName()).isEqualTo("application");
+		assertThat(env.getProfiles()).isEqualTo(new String[] { "default" });
+		assertThat(env.getLabel()).isEqualTo("master");
+		assertThat(env.getPropertySources()).isNotEmpty();
+		assertThat(env.getPropertySources().get(0).getSource().get("a.b")).isEqualTo("y");
+	}
+
+	@Configuration
+	protected static class ApplicationConfiguration {
+	}
+
+}

spring-cloud-config-server/src/test/resources/data-jdbc.sql

@@ -0,0 +1,2 @@
+INSERT into PROPERTIES(APPLICATION, PROFILE, LABEL, KEY, VALUE) values ('application', 'default', 'master', 'a.b', 'y');
+INSERT into PROPERTIES(APPLICATION, PROFILE, LABEL, KEY, VALUE) values ('foo', 'bar', 'master', 'a.b.c', 'x');

spring-cloud-config-server/src/test/resources/schema-jdbc.sql

@@ -0,0 +1,7 @@
+CREATE TABLE PROPERTIES (
+  KEY VARCHAR(2048),
+  VALUE VARCHAR(4096),
+  APPLICATION VARCHAR(128),
+  PROFILE VARCHAR(128),
+  LABEL VARCHAR(128)
+);