diff --git a/docs/src/main/asciidoc/spring-cloud-config.adoc b/docs/src/main/asciidoc/spring-cloud-config.adoc index f9f19e85..8ec76eb2 100644 --- a/docs/src/main/asciidoc/spring-cloud-config.adoc +++ b/docs/src/main/asciidoc/spring-cloud-config.adoc @@ -571,7 +571,7 @@ while providing tight access control and recording a detailed audit log. For more information on Vault see the https://www.vaultproject.io/intro/index.html[Vault quickstart guide]. -To enable the config server to use a Vault backend you must run your config server +To enable the config server to use a Vault backend you can run your config server with the `vault` profile. For example in your config server's `application.properties` you can add `spring.profiles.active=vault`. @@ -710,10 +710,30 @@ $ vault write secret/application foo=bar baz=bam All applications using the config server will have the properties `foo` and `baz` available to them. +==== JDBC Backend + +Spring Cloud Config Server supports JDBC (relation database) as a +backend for configuration properties. You can enable this feature by +adding `spring-jdbc` to the classpath, and using the "jdbc" profile, +or by adding a bean of type `JdbcEnvironmentRepository`. Spring Boot +will configure a data source if you include the right dependencies on +the classpath (see the user guide for more details on that). + +The database needs to have a table called "PROPERTIES" with columns +"APPLICATION", "PROFILE", "LABEL" (with the usual `Environment` +meaning), plus "KEY" and "VALUE" for the key and value pairs in +`Properties` style. All fields are of type String in Java, so you can +make them `VARCHAR` of whatever length you need. Property values +behave in the same way as they would if they came from Spring Boot +properties files named `{application}-{profile}.properties`, including +all the encryption and decryption, which will be applied as +post-processing steps (i.e. not in the repository implementation +directly). + ==== Composite Environment Repositories In some scenarios you may wish to pull configuration data from multiple -environment repositories. To do this just enable +environment repositories. To do this you can just enable multiple profiles in your config server's application properties or YAML file. If, for example, you want to pull configuration data from a Git repository as well as a SVN repository you would set the following properties for your diff --git a/pom.xml b/pom.xml index aca4a35f..96124fe9 100644 --- a/pom.xml +++ b/pom.xml @@ -11,7 +11,7 @@ org.springframework.cloud spring-cloud-build - 1.3.5.BUILD-SNAPSHOT + 1.3.5.RELEASE diff --git a/spring-cloud-config-server/pom.xml b/spring-cloud-config-server/pom.xml index 4fa6c4c8..922cc4fd 100644 --- a/spring-cloud-config-server/pom.xml +++ b/spring-cloud-config-server/pom.xml @@ -22,6 +22,11 @@ spring-boot-configuration-processor true + + org.springframework.boot + spring-boot-starter-jdbc + true + org.springframework.cloud spring-cloud-config-client @@ -60,6 +65,11 @@ aws-java-sdk-core true + + com.h2database + h2 + test + org.springframework.boot spring-boot-starter-test diff --git a/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/config/EnvironmentRepositoryConfiguration.java b/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/config/EnvironmentRepositoryConfiguration.java index d66fe444..b74e6b56 100644 --- a/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/config/EnvironmentRepositoryConfiguration.java +++ b/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/config/EnvironmentRepositoryConfiguration.java @@ -18,12 +18,14 @@ package org.springframework.cloud.config.server.config; import javax.servlet.http.HttpServletRequest; import org.eclipse.jgit.api.TransportConfigCallback; + import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean; import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty; import org.springframework.cloud.config.server.environment.ConsulEnvironmentWatch; import org.springframework.cloud.config.server.environment.EnvironmentRepository; import org.springframework.cloud.config.server.environment.EnvironmentWatch; +import org.springframework.cloud.config.server.environment.JdbcEnvironmentRepository; import org.springframework.cloud.config.server.environment.MultipleJGitEnvironmentRepository; import org.springframework.cloud.config.server.environment.NativeEnvironmentRepository; import org.springframework.cloud.config.server.environment.SvnKitEnvironmentRepository; @@ -33,6 +35,7 @@ import org.springframework.context.annotation.Configuration; import org.springframework.context.annotation.Import; import org.springframework.context.annotation.Profile; import org.springframework.core.env.ConfigurableEnvironment; +import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.web.client.RestTemplate; /** @@ -41,7 +44,7 @@ import org.springframework.web.client.RestTemplate; * */ @Configuration -@Import({ VaultRepositoryConfiguration.class, SvnRepositoryConfiguration.class, +@Import({ JdbcRepositoryConfiguration.class, VaultRepositoryConfiguration.class, SvnRepositoryConfiguration.class, NativeRepositoryConfiguration.class, GitRepositoryConfiguration.class, DefaultRepositoryConfiguration.class }) public class EnvironmentRepositoryConfiguration { @@ -147,3 +150,12 @@ class VaultRepositoryConfiguration { return new VaultEnvironmentRepository(request, watch, new RestTemplate()); } } + +@Configuration +@Profile("jdbc") +class JdbcRepositoryConfiguration { + @Bean + public JdbcEnvironmentRepository jdbcEnvironmentRepository(JdbcTemplate jdbc) { + return new JdbcEnvironmentRepository(jdbc); + } +} \ No newline at end of file diff --git a/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/JdbcEnvironmentRepository.java b/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/JdbcEnvironmentRepository.java new file mode 100644 index 00000000..f12c8687 --- /dev/null +++ b/spring-cloud-config-server/src/main/java/org/springframework/cloud/config/server/environment/JdbcEnvironmentRepository.java @@ -0,0 +1,123 @@ +/* + * Copyright 2016-2017 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.config.server.environment; + +import java.sql.ResultSet; +import java.sql.SQLException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Map; +import java.util.Properties; + +import org.springframework.boot.context.properties.ConfigurationProperties; +import org.springframework.cloud.config.environment.Environment; +import org.springframework.cloud.config.environment.PropertySource; +import org.springframework.core.Ordered; +import org.springframework.dao.DataAccessException; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.core.ResultSetExtractor; +import org.springframework.util.StringUtils; + +/** + * An {@link EnvironmentRepository} that picks up data from a relational database. The + * database should have a table called "PROPERTIES" with columns "APPLICATION", "PROFILE", + * "LABEL" (with the usual {@link Environment} meaning), plus "KEY" and "VALUE" for the + * key and value pairs in {@link Properties} style. Property values behave in the same way + * as they would if they came from Spring Boot properties files named + * {application}-{profile}.properties, including all the encryption and + * decryption, which will be applied as post-processing steps (i.e. not in this repository + * directly). + * + * @author Dave Syer + * + */ +@ConfigurationProperties("spring.cloud.config.server.jdbc") +public class JdbcEnvironmentRepository implements EnvironmentRepository, Ordered { + + private static final String DEFAULT_SQL = "SELECT KEY, VALUE from PROPERTIES where APPLICATION=? and PROFILE=? and LABEL=?"; + private int order = Ordered.LOWEST_PRECEDENCE - 10; + private final JdbcTemplate jdbc; + private String sql = DEFAULT_SQL; + private final PropertiesResultSetExtractor extractor = new PropertiesResultSetExtractor(); + + public JdbcEnvironmentRepository(JdbcTemplate jdbc) { + this.jdbc = jdbc; + } + + @Override + public Environment findOne(String application, String profile, String label) { + String config = application; + if (StringUtils.isEmpty(label)) { + label = "master"; + } + if (StringUtils.isEmpty(profile)) { + profile = "default"; + } + if (!profile.startsWith("default")) { + profile = "default," + profile; + } + String[] profiles = StringUtils.commaDelimitedListToStringArray(profile); + Environment environment = new Environment(application, profiles, label, null, + null); + if (!config.startsWith("application")) { + config = "application," + config; + } + List applications = new ArrayList(new LinkedHashSet<>( + Arrays.asList(StringUtils.commaDelimitedListToStringArray(config)))); + List envs = new ArrayList(new LinkedHashSet<>(Arrays.asList(profiles))); + Collections.reverse(applications); + Collections.reverse(envs); + for (String app : applications) { + for (String env : envs) { + Map next = (Map) jdbc.query(this.sql, + new Object[] { app, env, label }, this.extractor); + if (!next.isEmpty()) { + environment.add(new PropertySource(app + "-" + env, next)); + } + } + } + return environment; + } + + @Override + public int getOrder() { + return order; + } + + public void setOrder(int order) { + this.order = order; + } + +} + +class PropertiesResultSetExtractor implements ResultSetExtractor> { + + @Override + public Map extractData(ResultSet rs) + throws SQLException, DataAccessException { + Map map = new LinkedHashMap<>(); + while (rs.next()) { + map.put(rs.getString(1), rs.getString(2)); + } + return map; + } + +} \ No newline at end of file diff --git a/spring-cloud-config-server/src/test/java/org/springframework/cloud/config/server/AdhocTestSuite.java b/spring-cloud-config-server/src/test/java/org/springframework/cloud/config/server/AdhocTestSuite.java index 4749750f..7eb9188a 100644 --- a/spring-cloud-config-server/src/test/java/org/springframework/cloud/config/server/AdhocTestSuite.java +++ b/spring-cloud-config-server/src/test/java/org/springframework/cloud/config/server/AdhocTestSuite.java @@ -1,14 +1,44 @@ package org.springframework.cloud.config.server; -import org.junit.Ignore; import org.junit.runner.RunWith; import org.junit.runners.Suite; import org.junit.runners.Suite.SuiteClasses; + +import org.springframework.cloud.config.server.config.ConfigServerHealthIndicatorTests; +import org.springframework.cloud.config.server.config.CustomCompositeEnvironmentRepositoryTests; +import org.springframework.cloud.config.server.config.CustomEnvironmentRepositoryTests; +import org.springframework.cloud.config.server.config.TransportConfigurationTest; +import org.springframework.cloud.config.server.credentials.AwsCodeCommitCredentialsProviderTests; +import org.springframework.cloud.config.server.credentials.GitCredentialsProviderFactoryTests; +import org.springframework.cloud.config.server.encryption.CipherEnvironmentEncryptorTests; +import org.springframework.cloud.config.server.encryption.EncryptionControllerMultiTextEncryptorTests; +import org.springframework.cloud.config.server.encryption.EncryptionControllerTests; +import org.springframework.cloud.config.server.encryption.EncryptionIntegrationTests; +import org.springframework.cloud.config.server.encryption.EnvironmentPrefixHelperTests; +import org.springframework.cloud.config.server.encryption.KeyStoreTextEncryptorLocatorTests; +import org.springframework.cloud.config.server.environment.CompositeEnvironmentRepositoryTests; +import org.springframework.cloud.config.server.environment.EnvironmentControllerIntegrationTests; +import org.springframework.cloud.config.server.environment.EnvironmentControllerTests; import org.springframework.cloud.config.server.environment.EnvironmentEncryptorEnvironmentRepositoryTests; +import org.springframework.cloud.config.server.environment.JGitEnvironmentRepositoryConcurrencyTests; import org.springframework.cloud.config.server.environment.JGitEnvironmentRepositoryIntegrationTests; +import org.springframework.cloud.config.server.environment.JGitEnvironmentRepositoryTests; +import org.springframework.cloud.config.server.environment.JdbcEnvironmentRepositoryTests; +import org.springframework.cloud.config.server.environment.MultipleJGitEnvironmentApplicationPlaceholderRepositoryTests; +import org.springframework.cloud.config.server.environment.MultipleJGitEnvironmentLabelPlaceholderRepositoryTests; +import org.springframework.cloud.config.server.environment.MultipleJGitEnvironmentProfilePlaceholderRepositoryTests; import org.springframework.cloud.config.server.environment.MultipleJGitEnvironmentRepositoryIntegrationTests; +import org.springframework.cloud.config.server.environment.MultipleJGitEnvironmentRepositoryTests; import org.springframework.cloud.config.server.environment.NativeEnvironmentRepositoryTests; import org.springframework.cloud.config.server.environment.SVNKitEnvironmentRepositoryIntegrationTests; +import org.springframework.cloud.config.server.environment.SVNKitEnvironmentRepositoryTests; +import org.springframework.cloud.config.server.environment.VaultEnvironmentRepositoryTests; +import org.springframework.cloud.config.server.resource.GenericResourceRepositoryTests; +import org.springframework.cloud.config.server.resource.ResourceControllerIntegrationTests; +import org.springframework.cloud.config.server.resource.ResourceControllerTests; +import org.springframework.cloud.config.server.ssh.PropertyBasedSshSessionFactoryTest; +import org.springframework.cloud.config.server.ssh.SshPropertyValidatorTest; +import org.springframework.cloud.config.server.ssh.SshUriPropertyProcessorTest; /** * A test suite for probing weird ordering problems in the tests. @@ -16,10 +46,54 @@ import org.springframework.cloud.config.server.environment.SVNKitEnvironmentRepo * @author Dave Syer */ @RunWith(Suite.class) -@SuiteClasses({ MultipleJGitEnvironmentRepositoryIntegrationTests.class, - JGitEnvironmentRepositoryIntegrationTests.class, EnvironmentEncryptorEnvironmentRepositoryTests.class, - NativeEnvironmentRepositoryTests.class, SVNKitEnvironmentRepositoryIntegrationTests.class }) -@Ignore +@SuiteClasses({ TransportConfigurationIntegrationTests.PropertyBasedCallbackTest.class, + ConfigClientOnIntegrationTests.class, + BootstrapConfigServerIntegrationTests.class, + ResourceControllerIntegrationTests.class, + GenericResourceRepositoryTests.class, + ResourceControllerTests.class, + SubversionConfigServerIntegrationTests.class, + TransportConfigurationIntegrationTests.FileBasedCallbackTest.class, + TransportConfigurationTest.class, + ConfigServerHealthIndicatorTests.class, + CustomCompositeEnvironmentRepositoryTests.class, + CustomEnvironmentRepositoryTests.class, + ConfigClientOffIntegrationTests.class, + AwsCodeCommitCredentialsProviderTests.class, + GitCredentialsProviderFactoryTests.class, + PropertyBasedSshSessionFactoryTest.class, + SshUriPropertyProcessorTest.class, + SshPropertyValidatorTest.class, + NativeConfigServerIntegrationTests.class, + EncryptionIntegrationTests.ConfigSymmetricEncryptionIntegrationTests.class, + EnvironmentPrefixHelperTests.class, + EncryptionControllerTests.class, + CipherEnvironmentEncryptorTests.class, + EncryptionIntegrationTests.KeystoreConfigurationIntegrationTests.class, + KeyStoreTextEncryptorLocatorTests.class, + EncryptionIntegrationTests.BootstrapConfigSymmetricEncryptionIntegrationTests.class, + EncryptionControllerMultiTextEncryptorTests.class, + CompositeConfigServerIntegrationTests.class, + VanillaConfigServerIntegrationTests.class, + VaultEnvironmentRepositoryTests.class, + MultipleJGitEnvironmentLabelPlaceholderRepositoryTests.class, + EnvironmentControllerTests.class, + JGitEnvironmentRepositoryIntegrationTests.class, + CompositeEnvironmentRepositoryTests.class, + MultipleJGitEnvironmentApplicationPlaceholderRepositoryTests.class, + EnvironmentControllerIntegrationTests.class, + JGitEnvironmentRepositoryTests.class, + EnvironmentEncryptorEnvironmentRepositoryTests.class, + JdbcEnvironmentRepositoryTests.class, + SVNKitEnvironmentRepositoryTests.class, + MultipleJGitEnvironmentRepositoryTests.class, + NativeEnvironmentRepositoryTests.class, + JGitEnvironmentRepositoryConcurrencyTests.class, + SVNKitEnvironmentRepositoryIntegrationTests.class, + MultipleJGitEnvironmentRepositoryIntegrationTests.class, + MultipleJGitEnvironmentProfilePlaceholderRepositoryTests.class + }) +// @Ignore public class AdhocTestSuite { } diff --git a/spring-cloud-config-server/src/test/java/org/springframework/cloud/config/server/environment/EnvironmentControllerTests.java b/spring-cloud-config-server/src/test/java/org/springframework/cloud/config/server/environment/EnvironmentControllerTests.java index 4e492e4f..1eb3a85f 100644 --- a/spring-cloud-config-server/src/test/java/org/springframework/cloud/config/server/environment/EnvironmentControllerTests.java +++ b/spring-cloud-config-server/src/test/java/org/springframework/cloud/config/server/environment/EnvironmentControllerTests.java @@ -321,7 +321,6 @@ public class EnvironmentControllerTests { Mockito.when(this.repository.findOne("foo", "bar", null)) .thenReturn(this.environment); String json = this.controller.jsonProperties("foo", "bar", false).getBody(); - System.err.println(json); assertThat("Wrong output: " + json, json, is( "{\"a\":{\"b\":[{\"c\":\"x\",\"d\":[\"xx\",\"yy\"]},{\"c\":\"y\",\"e\":[{\"d\":\"z\"}]}]}}")); } diff --git a/spring-cloud-config-server/src/test/java/org/springframework/cloud/config/server/environment/JdbcEnvironmentRepositoryTests.java b/spring-cloud-config-server/src/test/java/org/springframework/cloud/config/server/environment/JdbcEnvironmentRepositoryTests.java new file mode 100644 index 00000000..d1ee7063 --- /dev/null +++ b/spring-cloud-config-server/src/test/java/org/springframework/cloud/config/server/environment/JdbcEnvironmentRepositoryTests.java @@ -0,0 +1,82 @@ +/* + * Copyright 2016-2017 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.config.server.environment; + +import javax.sql.DataSource; + +import org.junit.Test; +import org.junit.runner.RunWith; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.autoconfigure.jdbc.AutoConfigureTestDatabase; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.cloud.config.environment.Environment; +import org.springframework.cloud.config.server.environment.JdbcEnvironmentRepositoryTests.ApplicationConfiguration; +import org.springframework.context.annotation.Configuration; +import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.junit4.SpringRunner; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author Dave Syer + * + */ +@RunWith(SpringRunner.class) +@SpringBootTest(classes = ApplicationConfiguration.class, properties = { + "spring.datasource.schema=classpath:schema-jdbc.sql", + "spring.datasource.data=classpath:data-jdbc.sql" }) +@AutoConfigureTestDatabase +@DirtiesContext +public class JdbcEnvironmentRepositoryTests { + + @Autowired + private DataSource dataSource; + + @Test + public void basicProperties() { + Environment env = new JdbcEnvironmentRepository(new JdbcTemplate(dataSource)) + .findOne("foo", "bar", ""); + assertThat(env.getName()).isEqualTo("foo"); + assertThat(env.getProfiles()).isEqualTo(new String[] { "default", "bar" }); + assertThat(env.getLabel()).isEqualTo("master"); + assertThat(env.getPropertySources()).isNotEmpty(); + assertThat(env.getPropertySources().get(0).getName()).isEqualTo("foo-bar"); + assertThat(env.getPropertySources().get(0).getSource().get("a.b.c")) + .isEqualTo("x"); + assertThat(env.getPropertySources().get(1).getName()) + .isEqualTo("application-default"); + assertThat(env.getPropertySources().get(1).getSource().get("a.b")).isEqualTo("y"); + } + + @Test + public void defaults() { + Environment env = new JdbcEnvironmentRepository(new JdbcTemplate(dataSource)) + .findOne("application", "", ""); + assertThat(env.getName()).isEqualTo("application"); + assertThat(env.getProfiles()).isEqualTo(new String[] { "default" }); + assertThat(env.getLabel()).isEqualTo("master"); + assertThat(env.getPropertySources()).isNotEmpty(); + assertThat(env.getPropertySources().get(0).getSource().get("a.b")).isEqualTo("y"); + } + + @Configuration + protected static class ApplicationConfiguration { + } + +} diff --git a/spring-cloud-config-server/src/test/resources/data-jdbc.sql b/spring-cloud-config-server/src/test/resources/data-jdbc.sql new file mode 100644 index 00000000..adc78b5d --- /dev/null +++ b/spring-cloud-config-server/src/test/resources/data-jdbc.sql @@ -0,0 +1,2 @@ +INSERT into PROPERTIES(APPLICATION, PROFILE, LABEL, KEY, VALUE) values ('application', 'default', 'master', 'a.b', 'y'); +INSERT into PROPERTIES(APPLICATION, PROFILE, LABEL, KEY, VALUE) values ('foo', 'bar', 'master', 'a.b.c', 'x'); \ No newline at end of file diff --git a/spring-cloud-config-server/src/test/resources/schema-jdbc.sql b/spring-cloud-config-server/src/test/resources/schema-jdbc.sql new file mode 100644 index 00000000..5b714370 --- /dev/null +++ b/spring-cloud-config-server/src/test/resources/schema-jdbc.sql @@ -0,0 +1,7 @@ +CREATE TABLE PROPERTIES ( + KEY VARCHAR(2048), + VALUE VARCHAR(4096), + APPLICATION VARCHAR(128), + PROFILE VARCHAR(128), + LABEL VARCHAR(128) +); \ No newline at end of file