Added FILES configuration format.

Useful for tools like git2consul that load files into individual keys in consul KV.

Fixes gh-152

docs/src/main/asciidoc/spring-cloud-consul.adoc

@@ -185,6 +185,42 @@ You could store a YAML document in any of the keys listed above.
 
 You can change the data key using `spring.cloud.consul.config.data-key`.
 
+[[spring-cloud-consul-config-git2consul]]
+== git2consul with Config
+git2consul is a Consul community project that loads files from a git repository to individual keys into Consul. By default the names of the keys are names of the files. YAML and Properties files are supported with file extensions of `.yml` and `.properties` respectively.  Set the `spring.cloud.consul.config.format` property to `FILES`. For example:
+
+.bootstrap.yml
+----
+spring:
+  cloud:
+    consul:
+      config:
+        format: FILES
+----
+
+Given the following keys in `/config`, the `development` profile and an application name of `foo`:
+
+----
+.gitignore
+application.yml
+bar.properties
+foo-development.properties
+foo-production.yml
+foo.properties
+master.ref
+----
+
+the following property sources would be created:
+
+----
+config/foo-development.properties
+config/foo.properties
+config/application.yml
+----
+
+The value of each key needs to be a properly formatted YAML or Properties file.
+
+
 [[spring-cloud-consul-failfast]]
 == Fail Fast
 

spring-cloud-consul-config/src/main/java/org/springframework/cloud/consul/config/ConsulConfigProperties.java

@@ -16,6 +16,7 @@
 
 package org.springframework.cloud.consul.config;
 
+import javax.annotation.PostConstruct;
 import javax.validation.constraints.NotNull;
 
 import org.hibernate.validator.constraints.NotEmpty;
@@ -61,12 +62,18 @@ public class ConsulConfigProperties {
 	 */
 	private boolean failFast = true;
 
+	@PostConstruct
+	public void init() {
+		if (this.format == Format.FILES) {
+			this.profileSeparator = "-";
+		}
+	}
+
 	@Data
 	public class Watch {
 		private int waitTime = 2;
 		private boolean enabled = true;
 		private int delay = 10;
-
 	}
 
 	/**
@@ -109,7 +116,13 @@ public class ConsulConfigProperties {
 		 * Indicates that the configuration specified in consul is of YAML style i.e., value
 		 * of the consul key would be YAML format
 		 */
-		YAML;
+		YAML,
+
+		/**
+		 * Indicates that the configuration specified in consul uses keys as files.
+		 * This is useful for tools like git2consul.
+		 */
+		FILES,
 
 	}
 }

spring-cloud-consul-config/src/main/java/org/springframework/cloud/consul/config/ConsulFilesPropertySource.java

@@ -0,0 +1,47 @@
+/*
+ * Copyright 2013-2016 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.cloud.consul.config;
+
+import com.ecwid.consul.v1.ConsulClient;
+import com.ecwid.consul.v1.kv.model.GetValue;
+
+import static org.springframework.cloud.consul.config.ConsulConfigProperties.Format.PROPERTIES;
+import static org.springframework.cloud.consul.config.ConsulConfigProperties.Format.YAML;
+
+/**
+ * @author Spencer Gibb
+ */
+public class ConsulFilesPropertySource extends ConsulPropertySource {
+	public ConsulFilesPropertySource(String context, ConsulClient source, ConsulConfigProperties configProperties) {
+		super(context, source, configProperties);
+	}
+
+	@Override
+	public void init() {
+		//noop
+	}
+
+	public void init(GetValue value) {
+		if (this.getContext().endsWith(".yml")) {
+			parseValue(value, YAML);
+		} else if (this.getContext().endsWith(".properties")) {
+			parseValue(value, PROPERTIES);
+		} else {
+			throw new IllegalStateException("Unknown files extension for context " + this.getContext());
+		}
+	}
+}

spring-cloud-consul-config/src/main/java/org/springframework/cloud/consul/config/ConsulPropertySource.java

@@ -24,16 +24,18 @@ import java.util.Map;
 import java.util.Properties;
 import java.util.Set;
 
-import com.ecwid.consul.v1.ConsulClient;
-import com.ecwid.consul.v1.QueryParams;
-import com.ecwid.consul.v1.Response;
-import com.ecwid.consul.v1.kv.model.GetValue;
-
 import org.springframework.beans.factory.config.YamlPropertiesFactoryBean;
 import org.springframework.core.env.EnumerablePropertySource;
 import org.springframework.core.io.ByteArrayResource;
 import org.springframework.util.StringUtils;
 
+import com.ecwid.consul.v1.ConsulClient;
+import com.ecwid.consul.v1.QueryParams;
+import com.ecwid.consul.v1.Response;
+import com.ecwid.consul.v1.kv.model.GetValue;
+
+import static org.springframework.cloud.consul.config.ConsulConfigProperties.Format.PROPERTIES;
+import static org.springframework.cloud.consul.config.ConsulConfigProperties.Format.YAML;
 import static org.springframework.util.Base64Utils.decodeFromString;
 
 /**
@@ -52,12 +54,13 @@ public class ConsulPropertySource extends EnumerablePropertySource<ConsulClient>
 		this.context = context;
 		this.configProperties = configProperties;
 
-		if (!this.context.endsWith("/")) {
-			this.context = this.context + "/";
-		}
 	}
 
 	public void init() {
+		if (!this.context.endsWith("/")) {
+			this.context = this.context + "/";
+		}
+
 		Response<List<GetValue>> response = source.getKVValues(context,
 				configProperties.getAclToken(), QueryParams.DEFAULT);
 
@@ -79,7 +82,7 @@ public class ConsulPropertySource extends EnumerablePropertySource<ConsulClient>
 	 *
 	 * @param values
 	 */
-	private void parsePropertiesInKeyValueFormat(List<GetValue> values) {
+	protected void parsePropertiesInKeyValueFormat(List<GetValue> values) {
 		if (values == null) {
 			return;
 		}
@@ -100,7 +103,7 @@ public class ConsulPropertySource extends EnumerablePropertySource<ConsulClient>
 	 *
 	 * @param values
 	 */
-	private void parsePropertiesWithNonKeyValueFormat(List<GetValue> values,
+	protected void parsePropertiesWithNonKeyValueFormat(List<GetValue> values,
 			ConsulConfigProperties.Format format) {
 		if (values == null) {
 			return;
@@ -109,22 +112,26 @@ public class ConsulPropertySource extends EnumerablePropertySource<ConsulClient>
 		for (GetValue getValue : values) {
 			String key = getValue.getKey().replace(context, "");
 			if (configProperties.getDataKey().equals(key)) {
-				final String value = getDecoded(getValue.getValue());
-				final Properties props = generateProperties(value, format);
-
-				for (Map.Entry entry : props.entrySet()) {
-					properties
-							.put(entry.getKey().toString(), entry.getValue().toString());
-				}
+				parseValue(getValue, format);
 			}
 		}
 	}
 
-	private Properties generateProperties(String value,
+	protected void parseValue(GetValue getValue, ConsulConfigProperties.Format format) {
+		String value = getDecoded(getValue.getValue());
+		Properties props = generateProperties(value, format);
+
+		for (Map.Entry entry : props.entrySet()) {
+			properties
+					.put(entry.getKey().toString(), entry.getValue().toString());
+		}
+	}
+
+	protected Properties generateProperties(String value,
 			ConsulConfigProperties.Format format) {
 		final Properties props = new Properties();
 
-		if (format == ConsulConfigProperties.Format.PROPERTIES) {
+		if (format == PROPERTIES) {
 			try {
 				// Must use the ISO-8859-1 encoding because Properties.load(stream)
 				// expects it.
@@ -137,7 +144,7 @@ public class ConsulPropertySource extends EnumerablePropertySource<ConsulClient>
 
 			return props;
 		}
-		else if (format == ConsulConfigProperties.Format.YAML) {
+		else if (format == YAML) {
 			final YamlPropertiesFactoryBean yaml = new YamlPropertiesFactoryBean();
 			yaml.setResources(new ByteArrayResource(value.getBytes()));
 
@@ -153,6 +160,18 @@ public class ConsulPropertySource extends EnumerablePropertySource<ConsulClient>
 		return new String(decodeFromString(value));
 	}
 
+	protected Map<String, String> getProperties() {
+		return properties;
+	}
+
+	protected ConsulConfigProperties getConfigProperties() {
+		return configProperties;
+	}
+
+	protected String getContext() {
+		return context;
+	}
+
 	@Override
 	public Object getProperty(String name) {
 		return properties.get(name);

spring-cloud-consul-config/src/main/java/org/springframework/cloud/consul/config/ConsulPropertySourceLocator.java

@@ -31,9 +31,13 @@ import org.springframework.retry.annotation.Retryable;
 import org.springframework.util.ReflectionUtils;
 
 import com.ecwid.consul.v1.ConsulClient;
+import com.ecwid.consul.v1.Response;
+import com.ecwid.consul.v1.kv.model.GetValue;
 
 import lombok.extern.apachecommons.CommonsLog;
 
+import static org.springframework.cloud.consul.config.ConsulConfigProperties.Format.FILES;
+
 /**
  * @author Spencer Gibb
  */
@@ -66,23 +70,50 @@ public class ConsulPropertySourceLocator implements PropertySourceLocator {
 
 			String prefix = this.properties.getPrefix();
 
+			List<String> suffixes = new ArrayList<>();
+			if (this.properties.getFormat() != FILES) {
+				suffixes.add("/");
+			} else {
+				suffixes.add(".yml");
+				suffixes.add(".properties");
+			}
+
 			String defaultContext = prefix + "/" + this.properties.getDefaultContext();
-			this.contexts.add(defaultContext + "/");
-			addProfiles(this.contexts, defaultContext, profiles);
+			for (String suffix : suffixes) {
+				this.contexts.add(defaultContext + suffix);
+			}
+			for (String suffix : suffixes) {
+				addProfiles(this.contexts, defaultContext, profiles, suffix);
+			}
 
 			String baseContext = prefix + "/" + appName;
-			this.contexts.add(baseContext + "/");
-			addProfiles(this.contexts, baseContext, profiles);
-
-			CompositePropertySource composite = new CompositePropertySource("consul");
+			for (String suffix : suffixes) {
+				this.contexts.add(baseContext + suffix);
+			}
+			for (String suffix : suffixes) {
+				addProfiles(this.contexts, baseContext, profiles, suffix);
+			}
 
 			Collections.reverse(this.contexts);
 
+			CompositePropertySource composite = new CompositePropertySource("consul");
+
 			for (String propertySourceContext : this.contexts) {
 				try {
-					ConsulPropertySource propertySource = create(propertySourceContext);
-					propertySource.init();
-					composite.addPropertySource(propertySource);
+					ConsulPropertySource propertySource = null;
+					if (this.properties.getFormat() == FILES) {
+						Response<GetValue> response = this.consul.getKVValue(propertySourceContext, this.properties.getAclToken());
+						if (response.getValue() != null) {
+							ConsulFilesPropertySource filesPropertySource = new ConsulFilesPropertySource(propertySourceContext, this.consul, this.properties);
+							filesPropertySource.init(response.getValue());
+							propertySource = filesPropertySource;
+						}
+					} else {
+						propertySource = create(propertySourceContext);
+					}
+					if (propertySource != null) {
+						composite.addPropertySource(propertySource);
+					}
 				} catch (Exception e) {
 					if (this.properties.isFailFast()) {
 						ReflectionUtils.rethrowRuntimeException(e);
@@ -98,13 +129,15 @@ public class ConsulPropertySourceLocator implements PropertySourceLocator {
 	}
 
 	private ConsulPropertySource create(String context) {
-		return new ConsulPropertySource(context, consul, properties);
+		ConsulPropertySource propertySource = new ConsulPropertySource(context, this.consul, this.properties);
+		propertySource.init();
+		return propertySource;
 	}
 
 	private void addProfiles(List<String> contexts, String baseContext,
-			List<String> profiles) {
+							 List<String> profiles, String suffix) {
 		for (String profile : profiles) {
-			contexts.add(baseContext + this.properties.getProfileSeparator() + profile + "/");
+			contexts.add(baseContext + this.properties.getProfileSeparator() + profile + suffix);
 		}
 	}
 }

spring-cloud-consul-config/src/test/java/org/springframework/cloud/consul/config/ConsulPropertySourceLocatorFilesTests.java

@@ -0,0 +1,132 @@
+/*
+ * Copyright 2013-2016 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.cloud.consul.config;
+
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.UUID;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
+import org.springframework.boot.builder.SpringApplicationBuilder;
+import org.springframework.cloud.consul.ConsulProperties;
+import org.springframework.context.ConfigurableApplicationContext;
+import org.springframework.context.annotation.Configuration;
+import org.springframework.core.env.CompositePropertySource;
+import org.springframework.core.env.ConfigurableEnvironment;
+import org.springframework.core.env.MutablePropertySources;
+import org.springframework.core.env.PropertySource;
+import org.springframework.test.annotation.DirtiesContext;
+
+import com.ecwid.consul.v1.ConsulClient;
+
+import static org.hamcrest.Matchers.endsWith;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.hasSize;
+import static org.hamcrest.Matchers.is;
+import static org.hamcrest.Matchers.notNullValue;
+import static org.hamcrest.Matchers.typeCompatibleWith;
+import static org.junit.Assert.assertThat;
+
+/**
+ * @author Spencer Gibb
+ */
+@DirtiesContext
+public class ConsulPropertySourceLocatorFilesTests {
+	public static final String PREFIX = "_propertySourceLocatorFilesTests_config__";
+	public static final String ROOT = PREFIX + UUID.randomUUID();
+	public static final String APP_NAME = "testFilesFormat";
+	public static final String APPLICATION_YML = "/application.yml";
+	public static final String APPLICATION_DEV_YML = "/application-dev.yml";
+	public static final String APP_NAME_PROPS = "/"+APP_NAME+".properties";
+	public static final String APP_NAME_DEV_PROPS = "/"+APP_NAME+"-dev.properties";
+
+	private ConfigurableApplicationContext context;
+
+	@Configuration
+	@EnableAutoConfiguration
+	static class Config {
+
+	}
+
+	private ConfigurableEnvironment environment;
+	private ConsulClient client;
+	private ConsulProperties properties;
+
+	@Before
+	public void setup() {
+		this.properties = new ConsulProperties();
+		this.client = new ConsulClient(properties.getHost(), properties.getPort());
+		this.client.setKVValue(ROOT+ APPLICATION_YML, "foo: bar\nmy.baz: ${foo}");
+		this.client.setKVValue(ROOT+ APPLICATION_DEV_YML, "foo: bar-dev\nmy.baz: ${foo}");
+		this.client.setKVValue(ROOT+"/master.ref", UUID.randomUUID().toString());
+		this.client.setKVValue(ROOT+APP_NAME_PROPS, "foo: bar-app\nmy.baz: ${foo}");
+		this.client.setKVValue(ROOT+APP_NAME_DEV_PROPS, "foo: bar-app-dev\nmy.baz: ${foo}");
+
+		this.context = new SpringApplicationBuilder(Config.class)
+				.web(false)
+				.run("--spring.application.name="+ APP_NAME,
+						"--spring.cloud.consul.config.prefix="+ROOT,
+						"--spring.cloud.consul.config.format=FILES",
+						"--spring.profiles.active=dev",
+						"spring.cloud.consul.config.watch.delay=1");
+
+		this.client = context.getBean(ConsulClient.class);
+		this.properties = context.getBean(ConsulProperties.class);
+		this.environment = context.getEnvironment();
+	}
+
+	@After
+	public void teardown() {
+		this.client.deleteKVValues(PREFIX);
+		this.context.close();
+	}
+
+	@Test
+	public void propertySourcesFound() throws Exception {
+		String foo = this.environment.getProperty("foo");
+		assertThat("foo was wrong", foo, is(equalTo("bar-app-dev")));
+
+		String myBaz = this.environment.getProperty("my.baz");
+		assertThat("my.baz was wrong", myBaz, is(equalTo("bar-app-dev")));
+
+		MutablePropertySources propertySources = this.environment.getPropertySources();
+		PropertySource<?> bootstrapProperties = propertySources.get("bootstrapProperties");
+		assertThat("bootstrapProperties was null", bootstrapProperties, is(notNullValue()));
+		assertThat("bootstrapProperties was wrong type", bootstrapProperties.getClass(), is(typeCompatibleWith(CompositePropertySource.class)));
+
+		Collection<PropertySource<?>> consulSources = ((CompositePropertySource) bootstrapProperties).getPropertySources();
+		assertThat("consulSources was wrong size", consulSources, hasSize(1));
+
+		PropertySource<?> consulSource = consulSources.iterator().next();
+		assertThat("consulSource was wrong type", consulSource.getClass(), is(typeCompatibleWith(CompositePropertySource.class)));
+		Collection<PropertySource<?>> fileSources = ((CompositePropertySource) consulSource).getPropertySources();
+		assertThat("fileSources was wrong size", fileSources, hasSize(4));
+
+		assertFileSourceNames(fileSources, APP_NAME_DEV_PROPS, APP_NAME_PROPS, APPLICATION_DEV_YML, APPLICATION_YML);
+	}
+
+	private void assertFileSourceNames(Collection<PropertySource<?>> fileSources, String... names) {
+		Iterator<PropertySource<?>> iterator = fileSources.iterator();
+		for (String name : names) {
+			PropertySource<?> fileSource = iterator.next();
+			assertThat("fileSources was wrong name", fileSource.getName(), endsWith(name));
+		}
+	}
+}

spring-cloud-consul-config/src/test/java/org/springframework/cloud/consul/config/ConsulPropertySourceLocatorTests.java

@@ -36,6 +36,7 @@ import org.springframework.context.ConfigurableApplicationContext;
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 import org.springframework.core.env.ConfigurableEnvironment;
+import org.springframework.test.annotation.DirtiesContext;
 
 import static org.hamcrest.Matchers.equalTo;
 import static org.hamcrest.Matchers.is;
@@ -44,11 +45,18 @@ import static org.junit.Assert.assertThat;
 /**
  * @author Spencer Gibb
  */
+@DirtiesContext
 public class ConsulPropertySourceLocatorTests {
 	public static final String PREFIX = "_propertySourceLocatorTests_config__";
 	public static final String ROOT = PREFIX + UUID.randomUUID();
+	public static final String VALUE1 = "testPropVal";
+	public static final String TEST_PROP = "testProp";
+	public static final String KEY1 = ROOT + "/application/"+ TEST_PROP;
+	public static final String VALUE2 = "testPropVal2";
+	public static final String TEST_PROP2 = "testProp2";
+	public static final String KEY2 = ROOT + "/application/"+ TEST_PROP2;
+
 	private ConfigurableApplicationContext context;
-	public static final String KEY = ROOT + "/application/testProp";
 
 	@Configuration
 	@EnableAutoConfiguration
@@ -91,12 +99,13 @@ public class ConsulPropertySourceLocatorTests {
 		this.properties = new ConsulProperties();
 		this.client = new ConsulClient(properties.getHost(), properties.getPort());
 		this.client.deleteKVValues(PREFIX);
-		this.client.setKVValue(KEY, "testPropVal");
+		this.client.setKVValue(KEY1, VALUE1);
+		this.client.setKVValue(KEY2, VALUE2);
 
 
 		this.context = new SpringApplicationBuilder(Config.class)
 				.web(false)
-				.run("--spring.spring.application.name=testConsulPropertySourceLocator",
+				.run("--spring.application.name=testConsulPropertySourceLocator",
 						"--spring.cloud.consul.config.prefix="+ROOT,
 						"spring.cloud.consul.config.watch.delay=1");
 
@@ -108,21 +117,28 @@ public class ConsulPropertySourceLocatorTests {
 	@After
 	public void teardown() {
 		this.client.deleteKVValues(PREFIX);
+		this.context.close();
 	}
 
 	@Test
-	@Ignore // failing on travis
-	public void propertyLoadedAndUpdated() throws Exception {
-		String testProp = this.environment.getProperty("testProp");
-		assertThat("testProp was wrong", testProp, is(equalTo("testPropVal")));
+	public void propertyLoaded() throws Exception {
+		String testProp = this.environment.getProperty(TEST_PROP2);
+		assertThat("testProp was wrong", testProp, is(equalTo(VALUE2)));
+	}
 
-		this.client.setKVValue(KEY, "testPropValUpdate");
+	@Test
+	@Ignore("failing on travis")
+	public void propertyLoadedAndUpdated() throws Exception {
+		String testProp = this.environment.getProperty(TEST_PROP);
+		assertThat("testProp was wrong", testProp, is(equalTo(VALUE1)));
+
+		this.client.setKVValue(KEY1, "testPropValUpdate");
 
 		TestRefreshEndpoint endpoint = this.context.getBean(TestRefreshEndpoint.class);
 		boolean receivedEvent = endpoint.successLatch.await(3, TimeUnit.MINUTES);
 		assertThat("listener didn't receive event", receivedEvent, is(true));
 
-		testProp = this.environment.getProperty("testProp");
+		testProp = this.environment.getProperty(TEST_PROP);
 		assertThat("testProp was wrong after update", testProp, is(equalTo("testPropValUpdate")));
 
 		boolean receivedExtraEvent = endpoint.toManyLatch.await(15, TimeUnit.SECONDS);