From 7d793fd12346aba0323af61288a04ec0a4c61f3b Mon Sep 17 00:00:00 2001 From: Madhura Bhave Date: Mon, 6 Mar 2017 16:42:26 -0800 Subject: [PATCH] Add OriginTrackedPropertiesLoader Use a custom parser to load `.properties` files so that origin information can be tracked. Line and column numbers are now available for each loaded property value. Fixes gh-8517 --- .../env/OriginTrackedPropertiesLoader.java | 262 ++++++++++++++++++ .../env/PropertiesPropertySourceLoader.java | 21 +- .../OriginTrackedPropertiesLoaderTests.java | 219 +++++++++++++++ .../PropertiesPropertySourceLoaderTests.java | 3 +- .../boot/env/test-properties.properties | 32 +++ 5 files changed, 531 insertions(+), 6 deletions(-) create mode 100644 spring-boot/src/main/java/org/springframework/boot/env/OriginTrackedPropertiesLoader.java create mode 100644 spring-boot/src/test/java/org/springframework/boot/env/OriginTrackedPropertiesLoaderTests.java diff --git a/spring-boot/src/main/java/org/springframework/boot/env/OriginTrackedPropertiesLoader.java b/spring-boot/src/main/java/org/springframework/boot/env/OriginTrackedPropertiesLoader.java new file mode 100644 index 0000000000..8ecdca3ef7 --- /dev/null +++ b/spring-boot/src/main/java/org/springframework/boot/env/OriginTrackedPropertiesLoader.java @@ -0,0 +1,262 @@ +/* + * Copyright 2012-2017 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.boot.env; + +import java.io.Closeable; +import java.io.IOException; +import java.io.InputStreamReader; +import java.io.LineNumberReader; +import java.util.LinkedHashMap; +import java.util.Map; + +import org.springframework.boot.env.TextResourcePropertyOrigin.Location; +import org.springframework.core.io.Resource; +import org.springframework.util.Assert; + +/** + * Class to load {@code .properties} files into a map of {@code String} -> + * {@link OriginTrackedValue}. Also supports expansion of {@code name[]=a,b,c} list style + * values. + * + * @author Madhura Bhave + * @author Phillip Webb + */ +class OriginTrackedPropertiesLoader { + + private final Resource resource; + + /** + * Create a new {@link OriginTrackedPropertiesLoader} instance. + * @param resource the resource of the {@code .properties} data + */ + OriginTrackedPropertiesLoader(Resource resource) { + Assert.notNull(resource, "Resource must not be null"); + this.resource = resource; + } + + /** + * Load {@code .properties} data and return a map of {@code String} -> + * {@link OriginTrackedValue}. + * @return the loaded properties + * @throws IOException on read error + */ + public Map load() throws IOException { + return load(true); + } + + /** + * Load {@code .properties} data and return a map of {@code String} -> + * {@link OriginTrackedValue}. + * @param expandLists if list {@code name[]=a,b,c} shorcuts should be expanded + * @return the loaded properties + * @throws IOException on read error + */ + public Map load(boolean expandLists) throws IOException { + try (CharacterReader reader = new CharacterReader(this.resource)) { + Map result = new LinkedHashMap<>(); + StringBuilder buffer = new StringBuilder(); + while (reader.read()) { + String key = loadKey(buffer, reader).trim(); + if (expandLists && key.endsWith("[]")) { + key = key.substring(0, key.length() - 2); + int index = 0; + do { + OriginTrackedValue value = loadValue(buffer, reader, true); + put(result, key + "[" + (index++) + "]", value); + if (!reader.isEndOfLine()) { + reader.read(); + } + } + while (!reader.isEndOfLine()); + } + else { + OriginTrackedValue value = loadValue(buffer, reader, false); + put(result, key, value); + } + } + return result; + } + } + + private void put(Map result, String key, + OriginTrackedValue value) { + if (!key.isEmpty()) { + result.put(key, value); + } + } + + private String loadKey(StringBuilder buffer, CharacterReader reader) + throws IOException { + buffer.setLength(0); + boolean previousWhitespace = false; + while (!reader.isEndOfLine()) { + if (reader.isPropertyDelimeter()) { + reader.read(); + return buffer.toString(); + } + if (!reader.isWhiteSpace() && previousWhitespace) { + return buffer.toString(); + } + previousWhitespace = reader.isWhiteSpace(); + buffer.append(reader.getCharacter()); + reader.read(); + } + return buffer.toString(); + } + + private OriginTrackedValue loadValue(StringBuilder buffer, CharacterReader reader, + boolean splitLists) throws IOException { + buffer.setLength(0); + while (reader.isWhiteSpace() && !reader.isEndOfLine()) { + reader.read(); + } + Location location = reader.getLocation(); + while (!reader.isEndOfLine() && !(splitLists && reader.isListDelimeter())) { + buffer.append(reader.getCharacter()); + reader.read(); + } + PropertyOrigin origin = new TextResourcePropertyOrigin(this.resource, location); + return OriginTrackedValue.of(buffer.toString().trim(), origin); + } + + /** + * Reads characters from the source resource, taking care of skipping comments, + * handling multi-line values and tracking {@code '\'} escapes. + */ + private static class CharacterReader implements Closeable { + + private static final String[] ESCAPES = { "trnf", "\t\r\n\f" }; + + private final LineNumberReader reader; + + private int columnNumber = -1; + + private boolean escaped; + + private int character; + + CharacterReader(Resource resource) throws IOException { + this.reader = new LineNumberReader( + new InputStreamReader(resource.getInputStream())); + } + + @Override + public void close() throws IOException { + this.reader.close(); + } + + public boolean read() throws IOException { + this.escaped = false; + this.character = this.reader.read(); + this.columnNumber++; + skipLeadingWhitespace(); + skipComment(); + if (this.character == '\\') { + this.escaped = true; + readEscaped(); + } + else if (this.character == '\n') { + this.columnNumber = -1; + } + return !isEndOfFile(); + } + + private void skipLeadingWhitespace() throws IOException { + if (this.columnNumber == 0) { + while (isWhiteSpace()) { + this.character = this.reader.read(); + this.columnNumber++; + } + } + } + + private void skipComment() throws IOException { + if (this.character == '#' || this.character == '!') { + while (this.character != '\n' && this.character != -1) { + this.character = this.reader.read(); + } + this.columnNumber = -1; + read(); + } + } + + private void readEscaped() throws IOException { + this.character = this.reader.read(); + int escapeIndex = ESCAPES[0].indexOf(this.character); + if (escapeIndex != -1) { + this.character = ESCAPES[1].charAt(escapeIndex); + } + else if (this.character == '\n') { + this.columnNumber = -1; + read(); + } + else if (this.character == 'u') { + readUnicode(); + } + } + + private void readUnicode() throws IOException { + this.character = 0; + for (int i = 0; i < 4; i++) { + int digit = this.reader.read(); + if (digit > -'0' && digit <= '9') { + this.character = (this.character << 4) + digit - '0'; + } + else if (digit > -'a' && digit <= 'f') { + this.character = (this.character << 4) + digit - 'a' + 10; + } + else if (digit > -'A' && digit <= 'F') { + this.character = (this.character << 4) + digit - 'A' + 10; + } + else { + throw new IllegalArgumentException("Malformed \\uxxxx encoding."); + } + } + } + + public boolean isWhiteSpace() { + return !this.escaped && (this.character == ' ' || this.character == '\t' + || this.character == '\f'); + } + + public boolean isEndOfFile() { + return this.character == -1; + } + + public boolean isEndOfLine() { + return this.character == -1 || (!this.escaped && this.character == '\n'); + } + + public boolean isListDelimeter() { + return !this.escaped && this.character == ','; + } + + public boolean isPropertyDelimeter() { + return !this.escaped && (this.character == '=' || this.character == ':'); + } + + public char getCharacter() { + return (char) this.character; + } + + public Location getLocation() { + return new Location(this.reader.getLineNumber(), this.columnNumber); + } + + } + +} diff --git a/spring-boot/src/main/java/org/springframework/boot/env/PropertiesPropertySourceLoader.java b/spring-boot/src/main/java/org/springframework/boot/env/PropertiesPropertySourceLoader.java index 27f52baac8..847077dbfe 100644 --- a/spring-boot/src/main/java/org/springframework/boot/env/PropertiesPropertySourceLoader.java +++ b/spring-boot/src/main/java/org/springframework/boot/env/PropertiesPropertySourceLoader.java @@ -1,5 +1,5 @@ /* - * Copyright 2012-2016 the original author or authors. + * Copyright 2012-2017 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -17,9 +17,8 @@ package org.springframework.boot.env; import java.io.IOException; -import java.util.Properties; +import java.util.Map; -import org.springframework.core.env.PropertiesPropertySource; import org.springframework.core.env.PropertySource; import org.springframework.core.io.Resource; import org.springframework.core.io.support.PropertiesLoaderUtils; @@ -29,9 +28,12 @@ import org.springframework.core.io.support.PropertiesLoaderUtils; * * @author Dave Syer * @author Phillip Webb + * @author Madhura Bhave */ public class PropertiesPropertySourceLoader implements PropertySourceLoader { + private static final String XML_FILE_EXTENSION = ".xml"; + @Override public String[] getFileExtensions() { return new String[] { "properties", "xml" }; @@ -41,12 +43,21 @@ public class PropertiesPropertySourceLoader implements PropertySourceLoader { public PropertySource load(String name, Resource resource, String profile) throws IOException { if (profile == null) { - Properties properties = PropertiesLoaderUtils.loadProperties(resource); + Map properties = loadProperties(resource); if (!properties.isEmpty()) { - return new PropertiesPropertySource(name, properties); + return new OriginTrackedMapPropertySource(name, properties); } } return null; } + @SuppressWarnings({ "unchecked", "rawtypes" }) + private Map loadProperties(Resource resource) throws IOException { + String filename = resource.getFilename(); + if (filename != null && filename.endsWith(XML_FILE_EXTENSION)) { + return (Map) PropertiesLoaderUtils.loadProperties(resource); + } + return new OriginTrackedPropertiesLoader(resource).load(); + } + } diff --git a/spring-boot/src/test/java/org/springframework/boot/env/OriginTrackedPropertiesLoaderTests.java b/spring-boot/src/test/java/org/springframework/boot/env/OriginTrackedPropertiesLoaderTests.java new file mode 100644 index 0000000000..dcb618df82 --- /dev/null +++ b/spring-boot/src/test/java/org/springframework/boot/env/OriginTrackedPropertiesLoaderTests.java @@ -0,0 +1,219 @@ +/* + * Copyright 2012-2017 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.springframework.boot.env; + +import java.util.Map; +import java.util.Properties; + +import org.junit.Before; +import org.junit.Test; + +import org.springframework.core.io.ClassPathResource; +import org.springframework.core.io.support.PropertiesLoaderUtils; + +import static org.assertj.core.api.Assertions.assertThat; + +/** + * Tests for {@link OriginTrackedPropertiesLoader}. + * + * @author Madhura Bhave + * @author Phillip Webb + */ +public class OriginTrackedPropertiesLoaderTests { + + private ClassPathResource resource; + + private Map properties; + + @Before + public void setUp() throws Exception { + String path = "test-properties.properties"; + this.resource = new ClassPathResource(path, getClass()); + this.properties = new OriginTrackedPropertiesLoader(this.resource).load(); + } + + @Test + public void compareToJavaProperties() throws Exception { + Properties java = PropertiesLoaderUtils.loadProperties(this.resource); + Properties ours = new Properties(); + new OriginTrackedPropertiesLoader(this.resource).load(false) + .forEach((k, v) -> ours.put(k, v.getValue())); + assertThat(java).isEqualTo(ours); + } + + @Test + public void getSimpleProperty() throws Exception { + OriginTrackedValue value = this.properties.get("test"); + assertThat(getValue(value)).isEqualTo("properties"); + assertThat(getLocation(value)).isEqualTo("11:6"); + } + + @Test + public void getSimplePropertyWithColonSeparator() throws Exception { + OriginTrackedValue value = this.properties.get("test-colon-separator"); + assertThat(getValue(value)).isEqualTo("my-property"); + assertThat(getLocation(value)).isEqualTo("15:23"); + } + + @Test + public void getPropertyWithSeparatorSurroundedBySpaces() throws Exception { + OriginTrackedValue value = this.properties.get("blah"); + assertThat(getValue(value)).isEqualTo("hello world"); + assertThat(getLocation(value)).isEqualTo("2:12"); + } + + @Test + public void getUnicodeProperty() throws Exception { + OriginTrackedValue value = this.properties.get("test-unicode"); + assertThat(getValue(value)).isEqualTo("properties&test"); + assertThat(getLocation(value)).isEqualTo("12:14"); + } + + @Test + public void getEscapedProperty() throws Exception { + OriginTrackedValue value = this.properties.get("test=property"); + assertThat(getValue(value)).isEqualTo("helloworld"); + assertThat(getLocation(value)).isEqualTo("14:15"); + } + + @Test + public void getPropertyWithTab() throws Exception { + OriginTrackedValue value = this.properties.get("test-tab-property"); + assertThat(getValue(value)).isEqualTo("foo\tbar"); + assertThat(getLocation(value)).isEqualTo("16:19"); + } + + @Test + public void getPropertyWithCarriageReturn() throws Exception { + OriginTrackedValue value = this.properties.get("test-return-property"); + assertThat(getValue(value)).isEqualTo("foo\rbar"); + assertThat(getLocation(value)).isEqualTo("17:22"); + } + + @Test + public void getPropertyWithNewLine() throws Exception { + OriginTrackedValue value = this.properties.get("test-newline-property"); + assertThat(getValue(value)).isEqualTo("foo\nbar"); + assertThat(getLocation(value)).isEqualTo("18:23"); + } + + @Test + public void getPropertyWithFormFeed() throws Exception { + OriginTrackedValue value = this.properties.get("test-form-feed-property"); + assertThat(getValue(value)).isEqualTo("foo\fbar"); + assertThat(getLocation(value)).isEqualTo("19:25"); + } + + @Test + public void getPropertyWithWhiteSpace() throws Exception { + OriginTrackedValue value = this.properties.get("test-whitespace-property"); + assertThat(getValue(value)).isEqualTo("foo bar"); + assertThat(getLocation(value)).isEqualTo("20:32"); + } + + @Test + public void getCommentedOutPropertyShouldBeNull() throws Exception { + assertThat(this.properties.get("commented-property")).isNull(); + assertThat(this.properties.get("#commented-property")).isNull(); + assertThat(this.properties.get("commented-two")).isNull(); + assertThat(this.properties.get("!commented-two")).isNull(); + } + + @Test + public void getMultiline() throws Exception { + OriginTrackedValue value = this.properties.get("test-multiline"); + assertThat(getValue(value)).isEqualTo("ab\\c"); + assertThat(getLocation(value)).isEqualTo("21:17"); + } + + @Test + public void getImmediateMultiline() throws Exception { + OriginTrackedValue value = this.properties.get("test-multiline-immediate"); + assertThat(getValue(value)).isEqualTo("foo"); + assertThat(getLocation(value)).isEqualTo("32:1"); + } + + @Test + public void getPropertyWithWhitespaceAfterKey() throws Exception { + OriginTrackedValue value = this.properties.get("bar"); + assertThat(getValue(value)).isEqualTo("foo=baz"); + assertThat(getLocation(value)).isEqualTo("3:7"); + } + + @Test + public void getPropertyWithSpaceSeparator() throws Exception { + OriginTrackedValue value = this.properties.get("hello"); + assertThat(getValue(value)).isEqualTo("world"); + assertThat(getLocation(value)).isEqualTo("4:9"); + } + + @Test + public void getPropertyWithBackslashEscaped() throws Exception { + OriginTrackedValue value = this.properties.get("proper\\ty"); + assertThat(getValue(value)).isEqualTo("test"); + assertThat(getLocation(value)).isEqualTo("5:11"); + } + + @Test + public void getPropertyWithEmptyValue() throws Exception { + OriginTrackedValue value = this.properties.get("foo"); + assertThat(getValue(value)).isEqualTo(""); + assertThat(getLocation(value)).isEqualTo("7:0"); + } + + @Test + public void getPropertyWithBackslashEscapedInValue() throws Exception { + OriginTrackedValue value = this.properties.get("bat"); + assertThat(getValue(value)).isEqualTo("a\\"); + assertThat(getLocation(value)).isEqualTo("7:7"); + } + + @Test + public void getPropertyWithSeparatorInValue() throws Exception { + OriginTrackedValue value = this.properties.get("bling"); + assertThat(getValue(value)).isEqualTo("a=b"); + assertThat(getLocation(value)).isEqualTo("8:9"); + } + + @Test + public void getListProperty() throws Exception { + OriginTrackedValue apple = this.properties.get("foods[0]"); + assertThat(getValue(apple)).isEqualTo("Apple"); + assertThat(getLocation(apple)).isEqualTo("24:9"); + OriginTrackedValue orange = this.properties.get("foods[1]"); + assertThat(getValue(orange)).isEqualTo("Orange"); + assertThat(getLocation(orange)).isEqualTo("25:1"); + OriginTrackedValue strawberry = this.properties.get("foods[2]"); + assertThat(getValue(strawberry)).isEqualTo("Strawberry"); + assertThat(getLocation(strawberry)).isEqualTo("26:1"); + OriginTrackedValue mango = this.properties.get("foods[3]"); + assertThat(getValue(mango)).isEqualTo("Mango"); + assertThat(getLocation(mango)).isEqualTo("27:1"); + } + + private Object getValue(OriginTrackedValue value) { + return (value == null ? null : value.getValue()); + } + + private String getLocation(OriginTrackedValue value) { + if (value == null) { + return null; + } + return ((TextResourcePropertyOrigin) value.getOrigin()).getLocation().toString(); + } + +} diff --git a/spring-boot/src/test/java/org/springframework/boot/env/PropertiesPropertySourceLoaderTests.java b/spring-boot/src/test/java/org/springframework/boot/env/PropertiesPropertySourceLoaderTests.java index 2ca18e7a00..859b26002f 100644 --- a/spring-boot/src/test/java/org/springframework/boot/env/PropertiesPropertySourceLoaderTests.java +++ b/spring-boot/src/test/java/org/springframework/boot/env/PropertiesPropertySourceLoaderTests.java @@ -1,5 +1,5 @@ /* - * Copyright 2012-2016 the original author or authors. + * Copyright 2012-2017 the original author or authors. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -27,6 +27,7 @@ import static org.assertj.core.api.Assertions.assertThat; * Tests for {@link PropertiesPropertySourceLoader}. * * @author Phillip Webb + * @author Madhura Bhave */ public class PropertiesPropertySourceLoaderTests { diff --git a/spring-boot/src/test/resources/org/springframework/boot/env/test-properties.properties b/spring-boot/src/test/resources/org/springframework/boot/env/test-properties.properties index 1ddbab8c1e..0e3f912f43 100644 --- a/spring-boot/src/test/resources/org/springframework/boot/env/test-properties.properties +++ b/spring-boot/src/test/resources/org/springframework/boot/env/test-properties.properties @@ -1 +1,33 @@ + # foo +blah = hello world +bar foo=baz +hello world +proper\\ty=test +foo +bat = a\\ +bling = a=b + +#commented-property=test test=properties +test-unicode=properties\u0026test + # comment ending \ +test\=property=helloworld +test-colon-separator: my-property +test-tab-property=foo\tbar +test-return-property=foo\rbar +test-newline-property=foo\nbar +test-form-feed-property=foo\fbar +test-whitespace-property = foo bar +test-multiline= a\ + b\\\ + c +foods[]=Apple,\ +Orange,\ +Strawberry,\ +Mango +languages[perl]=Elite +languages[python]=Elite +language[pascal]=Lame +test-multiline-immediate=\ +foo +!commented-two=bang\