diff --git a/docs/src/main/asciidoc/spring-cloud-consul.adoc b/docs/src/main/asciidoc/spring-cloud-consul.adoc index 4f8fea0e..7f094838 100644 --- a/docs/src/main/asciidoc/spring-cloud-consul.adoc +++ b/docs/src/main/asciidoc/spring-cloud-consul.adoc @@ -185,6 +185,42 @@ You could store a YAML document in any of the keys listed above. You can change the data key using `spring.cloud.consul.config.data-key`. +[[spring-cloud-consul-config-git2consul]] +== git2consul with Config +git2consul is a Consul community project that loads files from a git repository to individual keys into Consul. By default the names of the keys are names of the files. YAML and Properties files are supported with file extensions of `.yml` and `.properties` respectively. Set the `spring.cloud.consul.config.format` property to `FILES`. For example: + +.bootstrap.yml +---- +spring: + cloud: + consul: + config: + format: FILES +---- + +Given the following keys in `/config`, the `development` profile and an application name of `foo`: + +---- +.gitignore +application.yml +bar.properties +foo-development.properties +foo-production.yml +foo.properties +master.ref +---- + +the following property sources would be created: + +---- +config/foo-development.properties +config/foo.properties +config/application.yml +---- + +The value of each key needs to be a properly formatted YAML or Properties file. + + [[spring-cloud-consul-failfast]] == Fail Fast diff --git a/spring-cloud-consul-config/src/main/java/org/springframework/cloud/consul/config/ConsulConfigProperties.java b/spring-cloud-consul-config/src/main/java/org/springframework/cloud/consul/config/ConsulConfigProperties.java index 47de704d..fd4b1a31 100644 --- a/spring-cloud-consul-config/src/main/java/org/springframework/cloud/consul/config/ConsulConfigProperties.java +++ b/spring-cloud-consul-config/src/main/java/org/springframework/cloud/consul/config/ConsulConfigProperties.java @@ -16,6 +16,7 @@ package org.springframework.cloud.consul.config; +import javax.annotation.PostConstruct; import javax.validation.constraints.NotNull; import org.hibernate.validator.constraints.NotEmpty; @@ -61,12 +62,18 @@ public class ConsulConfigProperties { */ private boolean failFast = true; + @PostConstruct + public void init() { + if (this.format == Format.FILES) { + this.profileSeparator = "-"; + } + } + @Data public class Watch { private int waitTime = 2; private boolean enabled = true; private int delay = 10; - } /** @@ -109,7 +116,13 @@ public class ConsulConfigProperties { * Indicates that the configuration specified in consul is of YAML style i.e., value * of the consul key would be YAML format */ - YAML; + YAML, + + /** + * Indicates that the configuration specified in consul uses keys as files. + * This is useful for tools like git2consul. + */ + FILES, } } diff --git a/spring-cloud-consul-config/src/main/java/org/springframework/cloud/consul/config/ConsulFilesPropertySource.java b/spring-cloud-consul-config/src/main/java/org/springframework/cloud/consul/config/ConsulFilesPropertySource.java new file mode 100644 index 00000000..8da169e5 --- /dev/null +++ b/spring-cloud-consul-config/src/main/java/org/springframework/cloud/consul/config/ConsulFilesPropertySource.java @@ -0,0 +1,47 @@ +/* + * Copyright 2013-2016 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.consul.config; + +import com.ecwid.consul.v1.ConsulClient; +import com.ecwid.consul.v1.kv.model.GetValue; + +import static org.springframework.cloud.consul.config.ConsulConfigProperties.Format.PROPERTIES; +import static org.springframework.cloud.consul.config.ConsulConfigProperties.Format.YAML; + +/** + * @author Spencer Gibb + */ +public class ConsulFilesPropertySource extends ConsulPropertySource { + public ConsulFilesPropertySource(String context, ConsulClient source, ConsulConfigProperties configProperties) { + super(context, source, configProperties); + } + + @Override + public void init() { + //noop + } + + public void init(GetValue value) { + if (this.getContext().endsWith(".yml")) { + parseValue(value, YAML); + } else if (this.getContext().endsWith(".properties")) { + parseValue(value, PROPERTIES); + } else { + throw new IllegalStateException("Unknown files extension for context " + this.getContext()); + } + } +} diff --git a/spring-cloud-consul-config/src/main/java/org/springframework/cloud/consul/config/ConsulPropertySource.java b/spring-cloud-consul-config/src/main/java/org/springframework/cloud/consul/config/ConsulPropertySource.java index 64987766..1dd30283 100644 --- a/spring-cloud-consul-config/src/main/java/org/springframework/cloud/consul/config/ConsulPropertySource.java +++ b/spring-cloud-consul-config/src/main/java/org/springframework/cloud/consul/config/ConsulPropertySource.java @@ -24,16 +24,18 @@ import java.util.Map; import java.util.Properties; import java.util.Set; -import com.ecwid.consul.v1.ConsulClient; -import com.ecwid.consul.v1.QueryParams; -import com.ecwid.consul.v1.Response; -import com.ecwid.consul.v1.kv.model.GetValue; - import org.springframework.beans.factory.config.YamlPropertiesFactoryBean; import org.springframework.core.env.EnumerablePropertySource; import org.springframework.core.io.ByteArrayResource; import org.springframework.util.StringUtils; +import com.ecwid.consul.v1.ConsulClient; +import com.ecwid.consul.v1.QueryParams; +import com.ecwid.consul.v1.Response; +import com.ecwid.consul.v1.kv.model.GetValue; + +import static org.springframework.cloud.consul.config.ConsulConfigProperties.Format.PROPERTIES; +import static org.springframework.cloud.consul.config.ConsulConfigProperties.Format.YAML; import static org.springframework.util.Base64Utils.decodeFromString; /** @@ -52,12 +54,13 @@ public class ConsulPropertySource extends EnumerablePropertySource this.context = context; this.configProperties = configProperties; - if (!this.context.endsWith("/")) { - this.context = this.context + "/"; - } } public void init() { + if (!this.context.endsWith("/")) { + this.context = this.context + "/"; + } + Response> response = source.getKVValues(context, configProperties.getAclToken(), QueryParams.DEFAULT); @@ -79,7 +82,7 @@ public class ConsulPropertySource extends EnumerablePropertySource * * @param values */ - private void parsePropertiesInKeyValueFormat(List values) { + protected void parsePropertiesInKeyValueFormat(List values) { if (values == null) { return; } @@ -100,7 +103,7 @@ public class ConsulPropertySource extends EnumerablePropertySource * * @param values */ - private void parsePropertiesWithNonKeyValueFormat(List values, + protected void parsePropertiesWithNonKeyValueFormat(List values, ConsulConfigProperties.Format format) { if (values == null) { return; @@ -109,22 +112,26 @@ public class ConsulPropertySource extends EnumerablePropertySource for (GetValue getValue : values) { String key = getValue.getKey().replace(context, ""); if (configProperties.getDataKey().equals(key)) { - final String value = getDecoded(getValue.getValue()); - final Properties props = generateProperties(value, format); - - for (Map.Entry entry : props.entrySet()) { - properties - .put(entry.getKey().toString(), entry.getValue().toString()); - } + parseValue(getValue, format); } } } - private Properties generateProperties(String value, + protected void parseValue(GetValue getValue, ConsulConfigProperties.Format format) { + String value = getDecoded(getValue.getValue()); + Properties props = generateProperties(value, format); + + for (Map.Entry entry : props.entrySet()) { + properties + .put(entry.getKey().toString(), entry.getValue().toString()); + } + } + + protected Properties generateProperties(String value, ConsulConfigProperties.Format format) { final Properties props = new Properties(); - if (format == ConsulConfigProperties.Format.PROPERTIES) { + if (format == PROPERTIES) { try { // Must use the ISO-8859-1 encoding because Properties.load(stream) // expects it. @@ -137,7 +144,7 @@ public class ConsulPropertySource extends EnumerablePropertySource return props; } - else if (format == ConsulConfigProperties.Format.YAML) { + else if (format == YAML) { final YamlPropertiesFactoryBean yaml = new YamlPropertiesFactoryBean(); yaml.setResources(new ByteArrayResource(value.getBytes())); @@ -153,6 +160,18 @@ public class ConsulPropertySource extends EnumerablePropertySource return new String(decodeFromString(value)); } + protected Map getProperties() { + return properties; + } + + protected ConsulConfigProperties getConfigProperties() { + return configProperties; + } + + protected String getContext() { + return context; + } + @Override public Object getProperty(String name) { return properties.get(name); diff --git a/spring-cloud-consul-config/src/main/java/org/springframework/cloud/consul/config/ConsulPropertySourceLocator.java b/spring-cloud-consul-config/src/main/java/org/springframework/cloud/consul/config/ConsulPropertySourceLocator.java index dfb07cc2..22333ca3 100644 --- a/spring-cloud-consul-config/src/main/java/org/springframework/cloud/consul/config/ConsulPropertySourceLocator.java +++ b/spring-cloud-consul-config/src/main/java/org/springframework/cloud/consul/config/ConsulPropertySourceLocator.java @@ -31,9 +31,13 @@ import org.springframework.retry.annotation.Retryable; import org.springframework.util.ReflectionUtils; import com.ecwid.consul.v1.ConsulClient; +import com.ecwid.consul.v1.Response; +import com.ecwid.consul.v1.kv.model.GetValue; import lombok.extern.apachecommons.CommonsLog; +import static org.springframework.cloud.consul.config.ConsulConfigProperties.Format.FILES; + /** * @author Spencer Gibb */ @@ -66,23 +70,50 @@ public class ConsulPropertySourceLocator implements PropertySourceLocator { String prefix = this.properties.getPrefix(); + List suffixes = new ArrayList<>(); + if (this.properties.getFormat() != FILES) { + suffixes.add("/"); + } else { + suffixes.add(".yml"); + suffixes.add(".properties"); + } + String defaultContext = prefix + "/" + this.properties.getDefaultContext(); - this.contexts.add(defaultContext + "/"); - addProfiles(this.contexts, defaultContext, profiles); + for (String suffix : suffixes) { + this.contexts.add(defaultContext + suffix); + } + for (String suffix : suffixes) { + addProfiles(this.contexts, defaultContext, profiles, suffix); + } String baseContext = prefix + "/" + appName; - this.contexts.add(baseContext + "/"); - addProfiles(this.contexts, baseContext, profiles); - - CompositePropertySource composite = new CompositePropertySource("consul"); + for (String suffix : suffixes) { + this.contexts.add(baseContext + suffix); + } + for (String suffix : suffixes) { + addProfiles(this.contexts, baseContext, profiles, suffix); + } Collections.reverse(this.contexts); + CompositePropertySource composite = new CompositePropertySource("consul"); + for (String propertySourceContext : this.contexts) { try { - ConsulPropertySource propertySource = create(propertySourceContext); - propertySource.init(); - composite.addPropertySource(propertySource); + ConsulPropertySource propertySource = null; + if (this.properties.getFormat() == FILES) { + Response response = this.consul.getKVValue(propertySourceContext, this.properties.getAclToken()); + if (response.getValue() != null) { + ConsulFilesPropertySource filesPropertySource = new ConsulFilesPropertySource(propertySourceContext, this.consul, this.properties); + filesPropertySource.init(response.getValue()); + propertySource = filesPropertySource; + } + } else { + propertySource = create(propertySourceContext); + } + if (propertySource != null) { + composite.addPropertySource(propertySource); + } } catch (Exception e) { if (this.properties.isFailFast()) { ReflectionUtils.rethrowRuntimeException(e); @@ -98,13 +129,15 @@ public class ConsulPropertySourceLocator implements PropertySourceLocator { } private ConsulPropertySource create(String context) { - return new ConsulPropertySource(context, consul, properties); + ConsulPropertySource propertySource = new ConsulPropertySource(context, this.consul, this.properties); + propertySource.init(); + return propertySource; } private void addProfiles(List contexts, String baseContext, - List profiles) { + List profiles, String suffix) { for (String profile : profiles) { - contexts.add(baseContext + this.properties.getProfileSeparator() + profile + "/"); + contexts.add(baseContext + this.properties.getProfileSeparator() + profile + suffix); } } } diff --git a/spring-cloud-consul-config/src/test/java/org/springframework/cloud/consul/config/ConsulPropertySourceLocatorFilesTests.java b/spring-cloud-consul-config/src/test/java/org/springframework/cloud/consul/config/ConsulPropertySourceLocatorFilesTests.java new file mode 100644 index 00000000..546ac62f --- /dev/null +++ b/spring-cloud-consul-config/src/test/java/org/springframework/cloud/consul/config/ConsulPropertySourceLocatorFilesTests.java @@ -0,0 +1,132 @@ +/* + * Copyright 2013-2016 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.consul.config; + +import java.util.Collection; +import java.util.Iterator; +import java.util.UUID; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.springframework.boot.autoconfigure.EnableAutoConfiguration; +import org.springframework.boot.builder.SpringApplicationBuilder; +import org.springframework.cloud.consul.ConsulProperties; +import org.springframework.context.ConfigurableApplicationContext; +import org.springframework.context.annotation.Configuration; +import org.springframework.core.env.CompositePropertySource; +import org.springframework.core.env.ConfigurableEnvironment; +import org.springframework.core.env.MutablePropertySources; +import org.springframework.core.env.PropertySource; +import org.springframework.test.annotation.DirtiesContext; + +import com.ecwid.consul.v1.ConsulClient; + +import static org.hamcrest.Matchers.endsWith; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.hasSize; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.typeCompatibleWith; +import static org.junit.Assert.assertThat; + +/** + * @author Spencer Gibb + */ +@DirtiesContext +public class ConsulPropertySourceLocatorFilesTests { + public static final String PREFIX = "_propertySourceLocatorFilesTests_config__"; + public static final String ROOT = PREFIX + UUID.randomUUID(); + public static final String APP_NAME = "testFilesFormat"; + public static final String APPLICATION_YML = "/application.yml"; + public static final String APPLICATION_DEV_YML = "/application-dev.yml"; + public static final String APP_NAME_PROPS = "/"+APP_NAME+".properties"; + public static final String APP_NAME_DEV_PROPS = "/"+APP_NAME+"-dev.properties"; + + private ConfigurableApplicationContext context; + + @Configuration + @EnableAutoConfiguration + static class Config { + + } + + private ConfigurableEnvironment environment; + private ConsulClient client; + private ConsulProperties properties; + + @Before + public void setup() { + this.properties = new ConsulProperties(); + this.client = new ConsulClient(properties.getHost(), properties.getPort()); + this.client.setKVValue(ROOT+ APPLICATION_YML, "foo: bar\nmy.baz: ${foo}"); + this.client.setKVValue(ROOT+ APPLICATION_DEV_YML, "foo: bar-dev\nmy.baz: ${foo}"); + this.client.setKVValue(ROOT+"/master.ref", UUID.randomUUID().toString()); + this.client.setKVValue(ROOT+APP_NAME_PROPS, "foo: bar-app\nmy.baz: ${foo}"); + this.client.setKVValue(ROOT+APP_NAME_DEV_PROPS, "foo: bar-app-dev\nmy.baz: ${foo}"); + + this.context = new SpringApplicationBuilder(Config.class) + .web(false) + .run("--spring.application.name="+ APP_NAME, + "--spring.cloud.consul.config.prefix="+ROOT, + "--spring.cloud.consul.config.format=FILES", + "--spring.profiles.active=dev", + "spring.cloud.consul.config.watch.delay=1"); + + this.client = context.getBean(ConsulClient.class); + this.properties = context.getBean(ConsulProperties.class); + this.environment = context.getEnvironment(); + } + + @After + public void teardown() { + this.client.deleteKVValues(PREFIX); + this.context.close(); + } + + @Test + public void propertySourcesFound() throws Exception { + String foo = this.environment.getProperty("foo"); + assertThat("foo was wrong", foo, is(equalTo("bar-app-dev"))); + + String myBaz = this.environment.getProperty("my.baz"); + assertThat("my.baz was wrong", myBaz, is(equalTo("bar-app-dev"))); + + MutablePropertySources propertySources = this.environment.getPropertySources(); + PropertySource bootstrapProperties = propertySources.get("bootstrapProperties"); + assertThat("bootstrapProperties was null", bootstrapProperties, is(notNullValue())); + assertThat("bootstrapProperties was wrong type", bootstrapProperties.getClass(), is(typeCompatibleWith(CompositePropertySource.class))); + + Collection> consulSources = ((CompositePropertySource) bootstrapProperties).getPropertySources(); + assertThat("consulSources was wrong size", consulSources, hasSize(1)); + + PropertySource consulSource = consulSources.iterator().next(); + assertThat("consulSource was wrong type", consulSource.getClass(), is(typeCompatibleWith(CompositePropertySource.class))); + Collection> fileSources = ((CompositePropertySource) consulSource).getPropertySources(); + assertThat("fileSources was wrong size", fileSources, hasSize(4)); + + assertFileSourceNames(fileSources, APP_NAME_DEV_PROPS, APP_NAME_PROPS, APPLICATION_DEV_YML, APPLICATION_YML); + } + + private void assertFileSourceNames(Collection> fileSources, String... names) { + Iterator> iterator = fileSources.iterator(); + for (String name : names) { + PropertySource fileSource = iterator.next(); + assertThat("fileSources was wrong name", fileSource.getName(), endsWith(name)); + } + } +} diff --git a/spring-cloud-consul-config/src/test/java/org/springframework/cloud/consul/config/ConsulPropertySourceLocatorTests.java b/spring-cloud-consul-config/src/test/java/org/springframework/cloud/consul/config/ConsulPropertySourceLocatorTests.java index 6fa7072b..c6978fca 100644 --- a/spring-cloud-consul-config/src/test/java/org/springframework/cloud/consul/config/ConsulPropertySourceLocatorTests.java +++ b/spring-cloud-consul-config/src/test/java/org/springframework/cloud/consul/config/ConsulPropertySourceLocatorTests.java @@ -36,6 +36,7 @@ import org.springframework.context.ConfigurableApplicationContext; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.core.env.ConfigurableEnvironment; +import org.springframework.test.annotation.DirtiesContext; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.is; @@ -44,11 +45,18 @@ import static org.junit.Assert.assertThat; /** * @author Spencer Gibb */ +@DirtiesContext public class ConsulPropertySourceLocatorTests { public static final String PREFIX = "_propertySourceLocatorTests_config__"; public static final String ROOT = PREFIX + UUID.randomUUID(); + public static final String VALUE1 = "testPropVal"; + public static final String TEST_PROP = "testProp"; + public static final String KEY1 = ROOT + "/application/"+ TEST_PROP; + public static final String VALUE2 = "testPropVal2"; + public static final String TEST_PROP2 = "testProp2"; + public static final String KEY2 = ROOT + "/application/"+ TEST_PROP2; + private ConfigurableApplicationContext context; - public static final String KEY = ROOT + "/application/testProp"; @Configuration @EnableAutoConfiguration @@ -91,12 +99,13 @@ public class ConsulPropertySourceLocatorTests { this.properties = new ConsulProperties(); this.client = new ConsulClient(properties.getHost(), properties.getPort()); this.client.deleteKVValues(PREFIX); - this.client.setKVValue(KEY, "testPropVal"); + this.client.setKVValue(KEY1, VALUE1); + this.client.setKVValue(KEY2, VALUE2); this.context = new SpringApplicationBuilder(Config.class) .web(false) - .run("--spring.spring.application.name=testConsulPropertySourceLocator", + .run("--spring.application.name=testConsulPropertySourceLocator", "--spring.cloud.consul.config.prefix="+ROOT, "spring.cloud.consul.config.watch.delay=1"); @@ -108,21 +117,28 @@ public class ConsulPropertySourceLocatorTests { @After public void teardown() { this.client.deleteKVValues(PREFIX); + this.context.close(); } @Test - @Ignore // failing on travis - public void propertyLoadedAndUpdated() throws Exception { - String testProp = this.environment.getProperty("testProp"); - assertThat("testProp was wrong", testProp, is(equalTo("testPropVal"))); + public void propertyLoaded() throws Exception { + String testProp = this.environment.getProperty(TEST_PROP2); + assertThat("testProp was wrong", testProp, is(equalTo(VALUE2))); + } - this.client.setKVValue(KEY, "testPropValUpdate"); + @Test + @Ignore("failing on travis") + public void propertyLoadedAndUpdated() throws Exception { + String testProp = this.environment.getProperty(TEST_PROP); + assertThat("testProp was wrong", testProp, is(equalTo(VALUE1))); + + this.client.setKVValue(KEY1, "testPropValUpdate"); TestRefreshEndpoint endpoint = this.context.getBean(TestRefreshEndpoint.class); boolean receivedEvent = endpoint.successLatch.await(3, TimeUnit.MINUTES); assertThat("listener didn't receive event", receivedEvent, is(true)); - testProp = this.environment.getProperty("testProp"); + testProp = this.environment.getProperty(TEST_PROP); assertThat("testProp was wrong after update", testProp, is(equalTo("testPropValUpdate"))); boolean receivedExtraEvent = endpoint.toManyLatch.await(15, TimeUnit.SECONDS);