Use a more compact layers.idx format

Update the `layers.idx` format so that it is more compact and can be
parsed by third-parties as YAML.

Closes gh-20860

spring-boot-project/spring-boot-tools/spring-boot-loader-tools/src/main/java/org/springframework/boot/loader/tools/LayersIndex.java

@@ -21,23 +21,46 @@ import java.io.IOException;
 import java.io.OutputStream;
 import java.io.OutputStreamWriter;
 import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
 import java.util.List;
+import java.util.Set;
 
 import org.springframework.util.LinkedMultiValueMap;
 import org.springframework.util.MultiValueMap;
 
 /**
- * Index describing the layer to which each entry in a jar belongs.
+ * Index describing the layer to which each entry in a jar belongs. Index files are simple
+ * text files that should be read from top to bottom. Each file defines the layers and
+ * their content. Layers names are written as quoted strings prefixed by a dash space
+ * ({@code "- "}) and with a colon ({@code ":"}) suffix. Layer content is either a file or
+ * folder name written as a quoted string prefixed by space space dash space
+ * ({@code "  - "}). A folder name ends with {@code /}, a file name does not. When a
+ * folder name is used it means that all files inside that folder are in the same layer.
+ * <p>
+ * Index files are designed to be compatible with YAML and may be read into a list of
+ * `Map&lt;String, List&lt;String&gt;&gt;` instances.
  *
  * @author Madhura Bhave
  * @author Andy Wilkinson
+ * @author Phillip Webb
  * @since 2.3.0
  */
 public class LayersIndex {
 
 	private final Iterable<Layer> layers;
 
-	private final MultiValueMap<Layer, String> index = new LinkedMultiValueMap<>();
+	private final Node root = new Node();
+
+	/**
+	 * Create a new {@link LayersIndex} backed by the given layers.
+	 * @param layers the layers in the index
+	 */
+	public LayersIndex(Layer... layers) {
+		this(Arrays.asList(layers));
+	}
 
 	/**
 	 * Create a new {@link LayersIndex} backed by the given layers.
@@ -53,7 +76,12 @@ public class LayersIndex {
 	 * @param name the name of the item
 	 */
 	public void add(Layer layer, String name) {
-		this.index.add(layer, name);
+		String[] segments = name.split("/");
+		Node node = this.root;
+		for (int i = 0; i < segments.length; i++) {
+			boolean isFolder = i < (segments.length - 1);
+			node = node.updateOrAddNode(segments[i], isFolder, layer);
+		}
 	}
 
 	/**
@@ -62,19 +90,67 @@ public class LayersIndex {
 	 * @throws IOException on IO error
 	 */
 	public void writeTo(OutputStream out) throws IOException {
+		MultiValueMap<Layer, String> index = new LinkedMultiValueMap<>();
+		this.root.buildIndex("", index);
+		index.values().forEach(Collections::sort);
 		BufferedWriter writer = new BufferedWriter(new OutputStreamWriter(out, StandardCharsets.UTF_8));
 		for (Layer layer : this.layers) {
-			List<String> names = this.index.get(layer);
-			if (names != null) {
+			List<String> names = index.get(layer);
+			if (names != null && !names.isEmpty()) {
+				writer.write("- \"" + layer + "\":\n");
 				for (String name : names) {
-					writer.write(layer.toString());
-					writer.write(" ");
-					writer.write(name);
-					writer.write("\n");
+					writer.write("  - \"" + name + "\"\n");
 				}
 			}
 		}
 		writer.flush();
+	}
+
+	/**
+	 * A node within the index represeting a single path segment.
+	 */
+	private static class Node {
+
+		private final String name;
+
+		private final Set<Layer> layers;
+
+		private final List<Node> children = new ArrayList<>();
+
+		Node() {
+			this.name = "";
+			this.layers = new HashSet<>();
+		}
+
+		Node(String name, Layer layer, Node parent) {
+			this.name = name;
+			this.layers = new HashSet<>(Collections.singleton(layer));
+		}
+
+		Node updateOrAddNode(String segment, boolean isFolder, Layer layer) {
+			String name = segment + (isFolder ? "/" : "");
+			for (Node child : this.children) {
+				if (name.equals(child.name)) {
+					child.layers.add(layer);
+					return child;
+				}
+			}
+			Node child = new Node(name, layer, this);
+			this.children.add(child);
+			return child;
+		}
+
+		void buildIndex(String path, MultiValueMap<Layer, String> index) {
+			String name = path + this.name;
+			if (this.layers.size() == 1) {
+				index.add(this.layers.iterator().next(), name);
+			}
+			else {
+				for (Node child : this.children) {
+					child.buildIndex(name, index);
+				}
+			}
+		}
 
 	}
 

spring-boot-project/spring-boot-tools/spring-boot-loader-tools/src/test/java/org/springframework/boot/loader/tools/AbstractPackagerTests.java

@@ -268,84 +268,20 @@ abstract class AbstractPackagerTests<P extends Packager> {
 		assertThat(hasPackagedEntry("BOOT-INF/layers.idx")).isTrue();
 		String layersIndex = getPackagedEntryContent("BOOT-INF/layers.idx");
 		List<String> expectedLayers = new ArrayList<>();
-		getExpectedLayers(expectedLayers);
-		expectedLayers.add("default " + "BOOT-INF/classpath.idx");
-		expectedLayers.add("default " + "BOOT-INF/layers.idx");
-		expectedLayers.add("0001 " + "BOOT-INF/lib/" + libJarFile1.getName());
-		expectedLayers.add("0002 " + "BOOT-INF/lib/" + libJarFile2.getName());
-		expectedLayers.add("0003 " + "BOOT-INF/lib/" + libJarFile3.getName());
-		assertThat(Arrays.asList(layersIndex.split("\\n"))).containsExactly(expectedLayers.toArray(new String[0]));
-	}
-
-	private void getExpectedLayers(List<String> expectedLayers) {
-		expectedLayers.add("default META-INF/MANIFEST.MF");
-		expectedLayers.add("default org/springframework/boot/loader/ClassPathIndexFile.class");
-		expectedLayers.add("default org/springframework/boot/loader/ExecutableArchiveLauncher.class");
-		expectedLayers.add("default org/springframework/boot/loader/JarLauncher.class");
-		expectedLayers.add(
-				"default org/springframework/boot/loader/LaunchedURLClassLoader$UseFastConnectionExceptionsEnumeration.class");
-		expectedLayers.add("default org/springframework/boot/loader/LaunchedURLClassLoader.class");
-		expectedLayers.add("default org/springframework/boot/loader/Launcher.class");
-		expectedLayers.add("default org/springframework/boot/loader/MainMethodRunner.class");
-		expectedLayers.add("default org/springframework/boot/loader/PropertiesLauncher$1.class");
-		expectedLayers.add("default org/springframework/boot/loader/PropertiesLauncher$ArchiveEntryFilter.class");
-		expectedLayers
-				.add("default org/springframework/boot/loader/PropertiesLauncher$PrefixMatchingArchiveFilter.class");
-		expectedLayers.add("default org/springframework/boot/loader/PropertiesLauncher.class");
-		expectedLayers.add("default org/springframework/boot/loader/WarLauncher.class");
-		expectedLayers.add("default org/springframework/boot/loader/archive/Archive$Entry.class");
-		expectedLayers.add("default org/springframework/boot/loader/archive/Archive$EntryFilter.class");
-		expectedLayers.add("default org/springframework/boot/loader/archive/Archive.class");
-		expectedLayers.add("default org/springframework/boot/loader/archive/ExplodedArchive$AbstractIterator.class");
-		expectedLayers.add("default org/springframework/boot/loader/archive/ExplodedArchive$ArchiveIterator.class");
-		expectedLayers.add("default org/springframework/boot/loader/archive/ExplodedArchive$EntryIterator.class");
-		expectedLayers.add("default org/springframework/boot/loader/archive/ExplodedArchive$FileEntry.class");
-		expectedLayers
-				.add("default org/springframework/boot/loader/archive/ExplodedArchive$SimpleJarFileArchive.class");
-		expectedLayers.add("default org/springframework/boot/loader/archive/ExplodedArchive.class");
-		expectedLayers.add("default org/springframework/boot/loader/archive/JarFileArchive$AbstractIterator.class");
-		expectedLayers.add("default org/springframework/boot/loader/archive/JarFileArchive$EntryIterator.class");
-		expectedLayers.add("default org/springframework/boot/loader/archive/JarFileArchive$JarFileEntry.class");
-		expectedLayers
-				.add("default org/springframework/boot/loader/archive/JarFileArchive$NestedArchiveIterator.class");
-		expectedLayers.add("default org/springframework/boot/loader/archive/JarFileArchive.class");
-		expectedLayers.add("default org/springframework/boot/loader/data/RandomAccessData.class");
-		expectedLayers.add("default org/springframework/boot/loader/data/RandomAccessDataFile$1.class");
-		expectedLayers.add("default org/springframework/boot/loader/data/RandomAccessDataFile$DataInputStream.class");
-		expectedLayers.add("default org/springframework/boot/loader/data/RandomAccessDataFile$FileAccess.class");
-		expectedLayers.add("default org/springframework/boot/loader/data/RandomAccessDataFile.class");
-		expectedLayers.add("default org/springframework/boot/loader/jar/AsciiBytes.class");
-		expectedLayers.add("default org/springframework/boot/loader/jar/Bytes.class");
-		expectedLayers.add("default org/springframework/boot/loader/jar/CentralDirectoryEndRecord$1.class");
-		expectedLayers.add("default org/springframework/boot/loader/jar/CentralDirectoryEndRecord$Zip64End.class");
-		expectedLayers.add("default org/springframework/boot/loader/jar/CentralDirectoryEndRecord$Zip64Locator.class");
-		expectedLayers.add("default org/springframework/boot/loader/jar/CentralDirectoryEndRecord.class");
-		expectedLayers.add("default org/springframework/boot/loader/jar/CentralDirectoryFileHeader.class");
-		expectedLayers.add("default org/springframework/boot/loader/jar/CentralDirectoryParser.class");
-		expectedLayers.add("default org/springframework/boot/loader/jar/CentralDirectoryVisitor.class");
-		expectedLayers.add("default org/springframework/boot/loader/jar/FileHeader.class");
-		expectedLayers.add("default org/springframework/boot/loader/jar/Handler.class");
-		expectedLayers.add("default org/springframework/boot/loader/jar/JarEntry.class");
-		expectedLayers.add("default org/springframework/boot/loader/jar/JarEntryFilter.class");
-		expectedLayers.add("default org/springframework/boot/loader/jar/JarFile$1.class");
-		expectedLayers.add("default org/springframework/boot/loader/jar/JarFile$2.class");
-		expectedLayers.add("default org/springframework/boot/loader/jar/JarFile$JarFileType.class");
-		expectedLayers.add("default org/springframework/boot/loader/jar/JarFile.class");
-		expectedLayers.add("default org/springframework/boot/loader/jar/JarFileEntries$1.class");
-		expectedLayers.add("default org/springframework/boot/loader/jar/JarFileEntries$EntryIterator.class");
-		expectedLayers.add("default org/springframework/boot/loader/jar/JarFileEntries.class");
-		expectedLayers.add("default org/springframework/boot/loader/jar/JarURLConnection$1.class");
-		expectedLayers.add("default org/springframework/boot/loader/jar/JarURLConnection$2.class");
-		expectedLayers.add("default org/springframework/boot/loader/jar/JarURLConnection$CloseAction.class");
-		expectedLayers.add("default org/springframework/boot/loader/jar/JarURLConnection$JarEntryName.class");
-		expectedLayers.add("default org/springframework/boot/loader/jar/JarURLConnection.class");
-		expectedLayers.add("default org/springframework/boot/loader/jar/StringSequence.class");
-		expectedLayers.add("default org/springframework/boot/loader/jar/ZipInflaterInputStream.class");
-		expectedLayers.add("default org/springframework/boot/loader/jarmode/JarMode.class");
-		expectedLayers.add("default org/springframework/boot/loader/jarmode/JarModeLauncher.class");
-		expectedLayers.add("default org/springframework/boot/loader/jarmode/TestJarMode.class");
-		expectedLayers.add("default org/springframework/boot/loader/util/SystemPropertyUtils.class");
-		expectedLayers.add("default BOOT-INF/classes/a/b/C.class");
+		expectedLayers.add("- 'default':");
+		expectedLayers.add("  - 'BOOT-INF/classes/'");
+		expectedLayers.add("  - 'BOOT-INF/classpath.idx'");
+		expectedLayers.add("  - 'BOOT-INF/layers.idx'");
+		expectedLayers.add("  - 'META-INF/'");
+		expectedLayers.add("  - 'org/'");
+		expectedLayers.add("- '0001':");
+		expectedLayers.add("  - 'BOOT-INF/lib/" + libJarFile1.getName() + "'");
+		expectedLayers.add("- '0002':");
+		expectedLayers.add("  - 'BOOT-INF/lib/" + libJarFile2.getName() + "'");
+		expectedLayers.add("- '0003':");
+		expectedLayers.add("  - 'BOOT-INF/lib/" + libJarFile3.getName() + "'");
+		assertThat(layersIndex.split("\\n"))
+				.containsExactly(expectedLayers.stream().map((s) -> s.replace('\'', '"')).toArray(String[]::new));
 	}
 
 	@Test
@@ -360,8 +296,13 @@ abstract class AbstractPackagerTests<P extends Packager> {
 		assertThat(Arrays.asList(classpathIndex.split("\\n"))).containsExactly("spring-boot-jarmode-layertools.jar");
 		assertThat(hasPackagedEntry("BOOT-INF/layers.idx")).isTrue();
 		String layersIndex = getPackagedEntryContent("BOOT-INF/layers.idx");
-		assertThat(Arrays.stream(layersIndex.split("\\n")).map((n) -> n.split(" ")[0]).distinct())
-				.containsExactly("default");
+		List<String> expectedLayers = new ArrayList<>();
+		expectedLayers.add("- 'default':");
+		expectedLayers.add("  - 'BOOT-INF/'");
+		expectedLayers.add("  - 'META-INF/'");
+		expectedLayers.add("  - 'org/'");
+		assertThat(layersIndex.split("\\n"))
+				.containsExactly(expectedLayers.stream().map((s) -> s.replace('\'', '"')).toArray(String[]::new));
 	}
 
 	@Test

spring-boot-project/spring-boot-tools/spring-boot-loader-tools/src/test/java/org/springframework/boot/loader/tools/LayersIndexTests.java

@@ -0,0 +1,143 @@
+/*
+ * Copyright 2012-2020 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.boot.loader.tools;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+
+import org.assertj.core.api.AbstractObjectAssert;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestInfo;
+
+import org.springframework.util.Assert;
+import org.springframework.util.FileCopyUtils;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * Tests for {@link LayersIndex}.
+ *
+ * @author Phillip Webb
+ */
+class LayersIndexTests {
+
+	private static final Layer LAYER_A = new Layer("a");
+
+	private static final Layer LAYER_B = new Layer("b");
+
+	private static final Layer LAYER_C = new Layer("c");
+
+	private String testMethodName;
+
+	@BeforeEach
+	void setup(TestInfo testInfo) {
+		this.testMethodName = testInfo.getTestMethod().get().getName();
+	}
+
+	@Test
+	void writeToWhenSimpleNamesSortsAlphabetically() throws Exception {
+		LayersIndex index = new LayersIndex(LAYER_A);
+		index.add(LAYER_A, "cat");
+		index.add(LAYER_A, "dog");
+		index.add(LAYER_A, "aardvark");
+		index.add(LAYER_A, "zerbra");
+		index.add(LAYER_A, "hamster");
+		assertThatIndex(index).writesExpectedContent();
+	}
+
+	@Test
+	void writeToWritesLayersInIteratorOrder() {
+		LayersIndex index = new LayersIndex(LAYER_B, LAYER_A, LAYER_C);
+		index.add(LAYER_A, "a1");
+		index.add(LAYER_A, "a2");
+		index.add(LAYER_B, "b1");
+		index.add(LAYER_B, "b2");
+		index.add(LAYER_C, "c1");
+		index.add(LAYER_C, "c2");
+		assertThatIndex(index).writesExpectedContent();
+	}
+
+	@Test
+	void writeToWhenLayerNotUsedSkipsLayer() {
+		LayersIndex index = new LayersIndex(LAYER_A, LAYER_B, LAYER_C);
+		index.add(LAYER_A, "a1");
+		index.add(LAYER_A, "a2");
+		index.add(LAYER_C, "c1");
+		index.add(LAYER_C, "c2");
+		assertThatIndex(index).writesExpectedContent();
+	}
+
+	@Test
+	void writeToWhenAllFilesInFolderAreInSameLayerUsesFolder() {
+		LayersIndex index = new LayersIndex(LAYER_A, LAYER_B, LAYER_C);
+		index.add(LAYER_A, "a1/b1/c1");
+		index.add(LAYER_A, "a1/b1/c2");
+		index.add(LAYER_A, "a1/b2/c1");
+		index.add(LAYER_B, "a2/b1");
+		index.add(LAYER_B, "a2/b2");
+		assertThatIndex(index).writesExpectedContent();
+	}
+
+	@Test
+	void writeToWhenAllFilesInFolderAreInNotInSameLayerUsesFiles() {
+		LayersIndex index = new LayersIndex(LAYER_A, LAYER_B, LAYER_C);
+		index.add(LAYER_A, "a1/b1/c1");
+		index.add(LAYER_B, "a1/b1/c2");
+		index.add(LAYER_C, "a1/b2/c1");
+		index.add(LAYER_A, "a2/b1");
+		index.add(LAYER_B, "a2/b2");
+		assertThatIndex(index).writesExpectedContent();
+
+	}
+
+	private LayersIndexAssert assertThatIndex(LayersIndex index) {
+		return new LayersIndexAssert(index);
+	}
+
+	private class LayersIndexAssert extends AbstractObjectAssert<LayersIndexAssert, LayersIndex> {
+
+		LayersIndexAssert(LayersIndex actual) {
+			super(actual, LayersIndexAssert.class);
+		}
+
+		void writesExpectedContent() {
+			try {
+				String actualContent = getContent();
+				String name = "LayersIndexTests-" + LayersIndexTests.this.testMethodName + ".txt";
+				InputStream in = LayersIndexTests.class.getResourceAsStream(name);
+				Assert.state(in != null, "Can't read " + name);
+				String expectedContent = new String(FileCopyUtils.copyToByteArray(in), StandardCharsets.UTF_8);
+				assertThat(actualContent).isEqualTo(expectedContent);
+			}
+			catch (IOException ex) {
+				throw new IllegalStateException(ex);
+			}
+
+		}
+
+		private String getContent() throws IOException {
+			ByteArrayOutputStream out = new ByteArrayOutputStream();
+			this.actual.writeTo(out);
+			return new String(out.toByteArray(), StandardCharsets.UTF_8);
+		}
+
+	}
+
+}

spring-boot-project/spring-boot-tools/spring-boot-loader-tools/src/test/resources/org/springframework/boot/loader/tools/LayersIndexTests-.txt

@@ -0,0 +1,6 @@
+- "a"
+  - "aardvark"
+  - "cat"
+  - "dog"
+  - "hamster"
+  - "zerbra"

spring-boot-project/spring-boot-tools/spring-boot-loader-tools/src/test/resources/org/springframework/boot/loader/tools/LayersIndexTests-writeToWhenAllFilesInFolderAreInNotInSameLayerUsesFiles.txt

@@ -0,0 +1,8 @@
+- "a":
+  - "a1/b1/c1"
+  - "a2/b1"
+- "b":
+  - "a1/b1/c2"
+  - "a2/b2"
+- "c":
+  - "a1/b2/"

spring-boot-project/spring-boot-tools/spring-boot-loader-tools/src/test/resources/org/springframework/boot/loader/tools/LayersIndexTests-writeToWhenAllFilesInFolderAreInSameLayerUsesFolder.txt

@@ -0,0 +1,4 @@
+- "a":
+  - "a1/"
+- "b":
+  - "a2/"

spring-boot-project/spring-boot-tools/spring-boot-loader-tools/src/test/resources/org/springframework/boot/loader/tools/LayersIndexTests-writeToWhenLayerNotUsedSkipsLayer.txt

@@ -0,0 +1,6 @@
+- "a":
+  - "a1"
+  - "a2"
+- "c":
+  - "c1"
+  - "c2"

spring-boot-project/spring-boot-tools/spring-boot-loader-tools/src/test/resources/org/springframework/boot/loader/tools/LayersIndexTests-writeToWhenSimpleNamesSortsAlphabetically.txt

@@ -0,0 +1,6 @@
+- "a":
+  - "aardvark"
+  - "cat"
+  - "dog"
+  - "hamster"
+  - "zerbra"

spring-boot-project/spring-boot-tools/spring-boot-loader-tools/src/test/resources/org/springframework/boot/loader/tools/LayersIndexTests-writeToWritesLayersInIteratorOrder.txt

@@ -0,0 +1,9 @@
+- "b":
+  - "b1"
+  - "b2"
+- "a":
+  - "a1"
+  - "a2"
+- "c":
+  - "c1"
+  - "c2"