From 1b31908eb428abb6d27180b549e636376f95e711 Mon Sep 17 00:00:00 2001 From: Srikalyan Swayampakula Date: Wed, 18 Nov 2015 16:56:42 -0800 Subject: [PATCH] Support different formats for config. Add support for YAML and Properties blobs. Key/Value is still the default. --- .../main/asciidoc/spring-cloud-consul.adoc | 27 ++++ .../consul/config/ConsulConfigProperties.java | 58 ++++++++- .../consul/config/ConsulPropertySource.java | 119 +++++++++++++++--- .../config/ConsulPropertySourceLocator.java | 2 +- .../config/ConsulPropertySourceTests.java | 106 ++++++++++++++++ 5 files changed, 289 insertions(+), 23 deletions(-) create mode 100644 spring-cloud-consul-config/src/test/java/org/springframework/cloud/consul/config/ConsulPropertySourceTests.java diff --git a/docs/src/main/asciidoc/spring-cloud-consul.adoc b/docs/src/main/asciidoc/spring-cloud-consul.adoc index 75462181..79542c1e 100644 --- a/docs/src/main/asciidoc/spring-cloud-consul.adoc +++ b/docs/src/main/asciidoc/spring-cloud-consul.adoc @@ -158,6 +158,33 @@ spring: * `profileSeparator` sets the value of the separator used to separate the profile name in property sources with profiles +[[spring-cloud-consul-config-format]] +== YAML or Properties with Config + +It may be more convenient to store a blob of properties in YAML or Properties format as opposed to individual key/value pairs. Set the `spring.cloud.consul.config.format` property to `YAML` or `PROPERTIES`. For example to use YAML: + +.bootstrap.yml +---- +spring: + cloud: + consul: + config: + format: YAML +---- + +YAML must be set in the appropriate `data` key in consul. Using the defaults above the keys would look like: + +---- +config/testApp,dev/data +config/testApp/data +config/application,dev/data +config/application/data +---- + +You could store a YAML document in any of the keys listed above. + +You can change the data key using `spring.cloud.consul.config.data-key`. + [[spring-cloud-consul-bus]] == Spring Cloud Bus with Consul diff --git a/spring-cloud-consul-config/src/main/java/org/springframework/cloud/consul/config/ConsulConfigProperties.java b/spring-cloud-consul-config/src/main/java/org/springframework/cloud/consul/config/ConsulConfigProperties.java index e202f8b7..d3297d89 100644 --- a/spring-cloud-consul-config/src/main/java/org/springframework/cloud/consul/config/ConsulConfigProperties.java +++ b/spring-cloud-consul-config/src/main/java/org/springframework/cloud/consul/config/ConsulConfigProperties.java @@ -16,11 +16,11 @@ package org.springframework.cloud.consul.config; -import lombok.Data; - import org.hibernate.validator.constraints.NotEmpty; import org.springframework.boot.context.properties.ConfigurationProperties; +import lombok.Data; + /** * @author Spencer Gibb */ @@ -38,5 +38,59 @@ public class ConsulConfigProperties { @NotEmpty private String profileSeparator = ","; + @NotEmpty + private Format format = Format.KEY_VALUE; + + /** + * If format is Format.PROPERTIES or Format.YAML + * then the following field is used as key to look up consul for configuration. + */ + @NotEmpty + private String dataKey = "data"; + private String aclToken; + + /** + * There are many ways in which we can specify configuration in consul i.e., + * + *
    + *
  1. + * Nested key value style: Where value is either a constant or part of the key (nested). + * For e.g., For following configuration a.b.c=something a.b.d=something else One can + * specify the configuration in consul with key as "../kv/config/application/a/b/c" and + * value as "something" and key as "../kv/config/application/a/b/d" and value as + * "something else"
    2. + *
  2. + * Entire contents of properties file as value For e.g., For following configuration + * a.b.c=something a.b.d=something else One can specify the configuration in consul with + * key as "../kv/config/application/properties" and value as whole configuration " + * a.b.c=something a.b.d=something else "
    3. + *
  3. + * as Json or YML. You get it.
    4. + *
+ * + * This enum specifies the different Formats/styles supported for loading the + * configuration. + * + * @author srikalyan.swayampakula + */ + public static enum Format { + /** + * Indicates that the configuration specified in consul is of type native key values. + */ + KEY_VALUE, + + /** + * Indicates that the configuration specified in consul is of property style i.e., + * value of the consul key would be a list of key=value pairs separated by new lines. + */ + PROPERTIES, + + /** + * Indicates that the configuration specified in consul is of YAML style i.e., value + * of the consul key would be YAML format + */ + YAML; + + } } diff --git a/spring-cloud-consul-config/src/main/java/org/springframework/cloud/consul/config/ConsulPropertySource.java b/spring-cloud-consul-config/src/main/java/org/springframework/cloud/consul/config/ConsulPropertySource.java index 27821e69..a29c6c2e 100644 --- a/spring-cloud-consul-config/src/main/java/org/springframework/cloud/consul/config/ConsulPropertySource.java +++ b/spring-cloud-consul-config/src/main/java/org/springframework/cloud/consul/config/ConsulPropertySource.java @@ -16,35 +16,41 @@ package org.springframework.cloud.consul.config; -import static org.springframework.util.Base64Utils.decodeFromString; - +import java.io.ByteArrayInputStream; +import java.io.IOException; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Properties; import java.util.Set; -import org.springframework.core.env.EnumerablePropertySource; -import org.springframework.util.StringUtils; - import com.ecwid.consul.v1.ConsulClient; import com.ecwid.consul.v1.QueryParams; import com.ecwid.consul.v1.Response; import com.ecwid.consul.v1.kv.model.GetValue; +import org.springframework.beans.factory.config.YamlPropertiesFactoryBean; +import org.springframework.core.env.EnumerablePropertySource; +import org.springframework.core.io.ByteArrayResource; +import org.springframework.util.StringUtils; + +import static org.springframework.util.Base64Utils.decodeFromString; + /** * @author Spencer Gibb */ public class ConsulPropertySource extends EnumerablePropertySource { private String context; - private String aclToken; + private ConsulConfigProperties configProperties; - private Map properties = new LinkedHashMap<>(); + private final Map properties = new LinkedHashMap<>(); - public ConsulPropertySource(String context, ConsulClient source, String aclToken) { + public ConsulPropertySource(String context, ConsulClient source, + ConsulConfigProperties configProperties) { super(context, source); this.context = context; - this.aclToken = aclToken; + this.configProperties = configProperties; if (!this.context.endsWith("/")) { this.context = this.context + "/"; @@ -53,25 +59,98 @@ public class ConsulPropertySource extends EnumerablePropertySource public void init() { Response> response; - if (aclToken == null) { + if (configProperties.getAclToken() == null) { response = source.getKVValues(context, QueryParams.DEFAULT); - } else { - response = source.getKVValues(context, aclToken, QueryParams.DEFAULT); } - List values = response.getValue(); + else { + response = source.getKVValues(context, configProperties.getAclToken(), + QueryParams.DEFAULT); + } - if (values != null) { - for (GetValue getValue : values) { - String key = getValue.getKey(); - if (!StringUtils.endsWithIgnoreCase(key, "/")) { - key = key.replace(context, "").replace('/', '.'); - String value = getDecoded(getValue.getValue()); - properties.put(key, value); + final List values = response.getValue(); + ConsulConfigProperties.Format format = configProperties.getFormat(); + switch (format) { + case KEY_VALUE: + parsePropertiesInKeyValueFormat(values); + break; + case PROPERTIES: + case YAML: + parsePropertiesWithNonKeyValueFormat(values, format); + } + } + + /** + * Parses the properties in key value style i.e., values are expected to be either a + * sub key or a constant + * + * @param values + */ + private void parsePropertiesInKeyValueFormat(List values) { + if (values == null) { + return; + } + + for (GetValue getValue : values) { + String key = getValue.getKey(); + if (!StringUtils.endsWithIgnoreCase(key, "/")) { + key = key.replace(context, "").replace('/', '.'); + String value = getDecoded(getValue.getValue()); + properties.put(key, value); + } + } + } + + /** + * Parses the properties using the format which is not a key value style i.e., either + * java properties style or YAML style + * + * @param values + */ + private void parsePropertiesWithNonKeyValueFormat(List values, + ConsulConfigProperties.Format format) { + if (values == null) { + return; + } + + for (GetValue getValue : values) { + String key = getValue.getKey().replace(context, ""); + if (configProperties.getDataKey().equals(key)) { + final String value = getDecoded(getValue.getValue()); + final Properties props = generateProperties(value, format); + + for (String propKey : props.stringPropertyNames()) { + properties.put(propKey, props.getProperty(propKey)); } } } } + private Properties generateProperties(String value, ConsulConfigProperties.Format format) { + final Properties props = new Properties(); + + if (format == ConsulConfigProperties.Format.PROPERTIES) { + try { + // Must use the ISO-8859-1 encoding because Properties.load(stream) + // expects it. + props.load(new ByteArrayInputStream(value.getBytes("ISO-8859-1"))); + } + catch (IOException e) { + throw new IllegalArgumentException(value + + " can't be encoded using ISO-8859-1"); + } + + return props; + } + else if (format == ConsulConfigProperties.Format.YAML) { + final YamlPropertiesFactoryBean yaml = new YamlPropertiesFactoryBean(); + yaml.setResources(new ByteArrayResource(value.getBytes())); + + return yaml.getObject(); + } + + return props; + } + public String getDecoded(String value) { if (value == null) return null; diff --git a/spring-cloud-consul-config/src/main/java/org/springframework/cloud/consul/config/ConsulPropertySourceLocator.java b/spring-cloud-consul-config/src/main/java/org/springframework/cloud/consul/config/ConsulPropertySourceLocator.java index 46a529b1..523484a3 100644 --- a/spring-cloud-consul-config/src/main/java/org/springframework/cloud/consul/config/ConsulPropertySourceLocator.java +++ b/spring-cloud-consul-config/src/main/java/org/springframework/cloud/consul/config/ConsulPropertySourceLocator.java @@ -77,7 +77,7 @@ public class ConsulPropertySourceLocator implements PropertySourceLocator { } private ConsulPropertySource create(String context) { - return new ConsulPropertySource(context, consul, properties.getAclToken()); + return new ConsulPropertySource(context, consul, properties); } private void addProfiles(List contexts, String baseContext, diff --git a/spring-cloud-consul-config/src/test/java/org/springframework/cloud/consul/config/ConsulPropertySourceTests.java b/spring-cloud-consul-config/src/test/java/org/springframework/cloud/consul/config/ConsulPropertySourceTests.java new file mode 100644 index 00000000..aaa4fdf7 --- /dev/null +++ b/spring-cloud-consul-config/src/test/java/org/springframework/cloud/consul/config/ConsulPropertySourceTests.java @@ -0,0 +1,106 @@ +/* + * Copyright 2013-2015 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.cloud.consul.config; + +import static org.hamcrest.Matchers.*; +import static org.junit.Assert.assertThat; + +import java.util.Random; + +import com.ecwid.consul.v1.ConsulClient; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.springframework.cloud.consul.ConsulProperties; + +/** + * @author Spencer Gibb + */ +public class ConsulPropertySourceTests { + + private ConsulClient client; + private ConsulProperties properties; + private String prefix; + private String kvContext; + private String propertiesContext; + + @Before + public void setup() { + properties = new ConsulProperties(); + prefix = "consulPropertySourceTests" + new Random().nextInt(Integer.MAX_VALUE); + properties.setPrefix(prefix); + client = new ConsulClient(properties.getHost(), properties.getPort()); + } + + @After + public void teardown() { + client.deleteKVValues(prefix); + } + + @Test + public void testKv() { + // key value properties + kvContext = prefix + "/kv"; + client.setKVValue(kvContext + "/fooprop", "fookvval"); + client.setKVValue(prefix+"/kv"+"/bar/prop", "barkvval"); + + ConsulPropertySource source = getConsulPropertySource(new ConsulConfigProperties(), kvContext); + + assertProperties(source, "fookvval", "barkvval"); + } + + private void assertProperties(ConsulPropertySource source, String fooval, String barval) { + assertThat("fooprop was wrong", (String)source.getProperty("fooprop"), is(equalTo(fooval))); + assertThat("bar.prop was wrong", (String)source.getProperty("bar.prop"), is(equalTo(barval))); + } + + @Test + public void testProperties() { + // properties file property + propertiesContext = prefix + "/properties"; + client.setKVValue(propertiesContext+"/data", "fooprop=foopropval\nbar.prop=barpropval"); + + ConsulConfigProperties configProperties = new ConsulConfigProperties(); + configProperties.setFormat(ConsulConfigProperties.Format.PROPERTIES); + ConsulPropertySource source = getConsulPropertySource(configProperties, propertiesContext); + + assertProperties(source, "foopropval", "barpropval"); + } + + @Test + public void testYaml() { + // yaml file property + String yamlContext = prefix + "/yaml"; + client.setKVValue(yamlContext+"/data", "fooprop: fooymlval\nbar:\n prop: barymlval"); + + ConsulConfigProperties configProperties = new ConsulConfigProperties(); + configProperties.setFormat(ConsulConfigProperties.Format.YAML); + ConsulPropertySource source = getConsulPropertySource(configProperties, yamlContext); + + assertProperties(source, "fooymlval", "barymlval"); + } + + private ConsulPropertySource getConsulPropertySource(ConsulConfigProperties configProperties, String context) { + ConsulPropertySource source = new ConsulPropertySource(context, client, configProperties); + source.init(); + String[] names = source.getPropertyNames(); + assertThat("names was null", names, is(notNullValue())); + assertThat("names was wrong size", names.length, is(equalTo(2))); + return source; + } +}