Support gzip compressed image layers

Update buildpack support to allow gzip compressed image layers to be
used when returned by the Docker engine. This update is restores
buildpack support when using Docker Desktop with the "Use containerd
for pulling and storing images" option enabled.

This commit introduces a new `ExportedImageTar` class to deal with the
intricacies of determining the mimetype of a layer. The class deals with
the parsing of `index.json' and related manifest blobs in order to
obtain layer information. The legacy `manifest.json` format is also
supported should `index.json` be missing.

Tests have been added to ensure that export archives from Docker Engine,
Docker Desktop (with and without containerd), and Podman can be used.

Fixes gh-40100

Co-authored-by: Moritz Halbritter <moritz.halbritter@broadcom.com>
Co-authored-by: Scott Frederick <scott.frederick@broadcom.com>

spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/test/java/org/springframework/boot/buildpack/platform/build/ImageBuildpackTests.java

@@ -19,10 +19,10 @@ package org.springframework.boot.buildpack.platform.build;
 import java.io.ByteArrayInputStream;
 import java.io.ByteArrayOutputStream;
 import java.io.File;
+import java.io.FileInputStream;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.nio.file.Files;
-import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Random;
@@ -37,6 +37,8 @@ import org.mockito.invocation.InvocationOnMock;
 import org.springframework.boot.buildpack.platform.docker.type.Image;
 import org.springframework.boot.buildpack.platform.docker.type.ImageReference;
 import org.springframework.boot.buildpack.platform.io.IOBiConsumer;
+import org.springframework.boot.buildpack.platform.io.TarArchive;
+import org.springframework.boot.buildpack.platform.io.TarArchive.Compression;
 import org.springframework.boot.buildpack.platform.json.AbstractJsonTests;
 
 import static org.assertj.core.api.Assertions.assertThat;
@@ -176,7 +178,7 @@ class ImageBuildpackTests extends AbstractJsonTests {
 
 	private Object withMockLayers(InvocationOnMock invocation) {
 		try {
-			IOBiConsumer<String, Path> consumer = invocation.getArgument(1);
+			IOBiConsumer<String, TarArchive> consumer = invocation.getArgument(1);
 			File tarFile = File.createTempFile("create-builder-test-", null);
 			FileOutputStream out = new FileOutputStream(tarFile);
 			try (TarArchiveOutputStream tarOut = new TarArchiveOutputStream(out)) {
@@ -189,7 +191,7 @@ class ImageBuildpackTests extends AbstractJsonTests {
 				writeTarEntry(tarOut, "/cnb/buildpacks/example_buildpack/0.0.1/" + this.longFilePath);
 				tarOut.finish();
 			}
-			consumer.accept("test", tarFile.toPath());
+			consumer.accept("test", TarArchive.fromInputStream(new FileInputStream(tarFile), Compression.NONE));
 			Files.delete(tarFile.toPath());
 		}
 		catch (IOException ex) {

spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/test/java/org/springframework/boot/buildpack/platform/docker/DockerApiTests.java

@@ -313,12 +313,14 @@ class DockerApiTests {
 		}
 
 		@Test
+		@SuppressWarnings("removal")
 		void exportLayersWhenReferenceIsNullThrowsException() {
 			assertThatIllegalArgumentException().isThrownBy(() -> this.api.exportLayerFiles(null, (name, archive) -> {
 			})).withMessage("Reference must not be null");
 		}
 
 		@Test
+		@SuppressWarnings("removal")
 		void exportLayersWhenExportsIsNullThrowsException() {
 			ImageReference reference = ImageReference.of("gcr.io/paketo-buildpacks/builder:base");
 			assertThatIllegalArgumentException().isThrownBy(() -> this.api.exportLayerFiles(reference, null))
@@ -393,14 +395,15 @@ class DockerApiTests {
 		}
 
 		@Test
+		@SuppressWarnings("removal")
 		void exportLayersWithNoManifestThrowsException() throws Exception {
 			ImageReference reference = ImageReference.of("gcr.io/paketo-buildpacks/builder:base");
 			URI exportUri = new URI(IMAGES_URL + "/gcr.io/paketo-buildpacks/builder:base/get");
 			given(DockerApiTests.this.http.get(exportUri)).willReturn(responseOf("export-no-manifest.tar"));
-			assertThatIllegalArgumentException()
-				.isThrownBy(() -> this.api.exportLayerFiles(reference, (name, archive) -> {
-				}))
-				.withMessageContaining("Manifest not found in image " + reference);
+			String expectedMessage = "Exported image '%s' does not contain 'index.json' or 'manifest.json'"
+				.formatted(reference);
+			assertThatIllegalStateException().isThrownBy(() -> this.api.exportLayerFiles(reference, (name, archive) -> {
+			})).withMessageContaining(expectedMessage);
 		}
 
 		@Test

spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/test/java/org/springframework/boot/buildpack/platform/docker/ExportedImageTarTests.java

@@ -0,0 +1,53 @@
+/*
+ * Copyright 2012-2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.boot.buildpack.platform.docker;
+
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import org.springframework.boot.buildpack.platform.docker.type.ImageReference;
+import org.springframework.boot.buildpack.platform.io.TarArchive.Compression;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * Tests for {@link ExportedImageTar}.
+ *
+ * @author Phillip Webb
+ * @author Scott Frederick
+ */
+class ExportedImageTarTests {
+
+	@ParameterizedTest
+	@ValueSource(strings = { "export-docker-desktop.tar", "export-docker-desktop-containerd.tar",
+			"export-docker-desktop-containerd-manifest-list.tar", "export-docker-engine.tar", "export-podman.tar" })
+	void test(String tarFile) throws Exception {
+		ImageReference reference = ImageReference.of("test:latest");
+		try (ExportedImageTar exportedImageTar = new ExportedImageTar(reference,
+				getClass().getResourceAsStream(tarFile))) {
+			Compression expectedCompression = (!tarFile.contains("containerd")) ? Compression.NONE : Compression.GZIP;
+			String expectedName = (expectedCompression != Compression.GZIP)
+					? "5caae51697b248b905dca1a4160864b0e1a15c300981736555cdce6567e8d477"
+					: "f0f1fd1bdc71ac6a4dc99cea5f5e45c86c5ec26fe4d1daceeb78207303606429";
+			exportedImageTar.exportLayers((name, tarArchive) -> {
+				assertThat(name).contains(expectedName);
+				assertThat(tarArchive.getCompression()).isEqualTo(expectedCompression);
+			});
+		}
+	}
+
+}

spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/test/java/org/springframework/boot/buildpack/platform/docker/type/ImageArchiveIndexTests.java

@@ -0,0 +1,50 @@
+/*
+ * Copyright 2012-2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.boot.buildpack.platform.docker.type;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.Test;
+
+import org.springframework.boot.buildpack.platform.json.AbstractJsonTests;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * Tests for {@link ImageArchiveIndex}.
+ *
+ * @author Phillip Webb
+ */
+class ImageArchiveIndexTests extends AbstractJsonTests {
+
+	@Test
+	void loadJson() throws IOException {
+		String content = getContentAsString("image-archive-index.json");
+		ImageArchiveIndex index = getIndex(content);
+		assertThat(index.getSchemaVersion()).isEqualTo(2);
+		assertThat(index.getManifests()).hasSize(1);
+		BlobReference manifest = index.getManifests().get(0);
+		assertThat(manifest.getMediaType()).isEqualTo("application/vnd.docker.distribution.manifest.list.v2+json");
+		assertThat(manifest.getDigest())
+			.isEqualTo("sha256:3bbe02431d8e5124ffe816ec27bf6508b50edd1d10218be1a03e799a186b9004");
+	}
+
+	private ImageArchiveIndex getIndex(String content) throws IOException {
+		return new ImageArchiveIndex(getObjectMapper().readTree(content));
+	}
+
+}

spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/test/java/org/springframework/boot/buildpack/platform/docker/type/ImageArchiveManifestTests.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2012-2023 the original author or authors.
+ * Copyright 2012-2024 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -36,7 +36,8 @@ class ImageArchiveManifestTests extends AbstractJsonTests {
 
 	@Test
 	void getLayersReturnsLayers() throws Exception {
-		ImageArchiveManifest manifest = getManifest();
+		String content = getContentAsString("image-archive-manifest.json");
+		ImageArchiveManifest manifest = getManifest(content);
 		List<String> expectedLayers = new ArrayList<>();
 		for (int blankLayersCount = 0; blankLayersCount < 46; blankLayersCount++) {
 			expectedLayers.add("blank_" + blankLayersCount);
@@ -50,7 +51,7 @@ class ImageArchiveManifestTests extends AbstractJsonTests {
 	@Test
 	void getLayersWithNoLayersReturnsEmptyList() throws Exception {
 		String content = "[{\"Layers\": []}]";
-		ImageArchiveManifest manifest = new ImageArchiveManifest(getObjectMapper().readTree(content));
+		ImageArchiveManifest manifest = getManifest(content);
 		assertThat(manifest.getEntries()).hasSize(1);
 		assertThat(manifest.getEntries().get(0).getLayers()).isEmpty();
 	}
@@ -58,12 +59,12 @@ class ImageArchiveManifestTests extends AbstractJsonTests {
 	@Test
 	void getLayersWithEmptyManifestReturnsEmptyList() throws Exception {
 		String content = "[]";
-		ImageArchiveManifest manifest = new ImageArchiveManifest(getObjectMapper().readTree(content));
+		ImageArchiveManifest manifest = getManifest(content);
 		assertThat(manifest.getEntries()).isEmpty();
 	}
 
-	private ImageArchiveManifest getManifest() throws IOException {
-		return new ImageArchiveManifest(getObjectMapper().readTree(getContent("image-archive-manifest.json")));
+	private ImageArchiveManifest getManifest(String content) throws IOException {
+		return new ImageArchiveManifest(getObjectMapper().readTree(content));
 	}
 
 }

spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/test/java/org/springframework/boot/buildpack/platform/docker/type/ManifestListTests.java

@@ -0,0 +1,47 @@
+/*
+ * Copyright 2012-2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.boot.buildpack.platform.docker.type;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.Test;
+
+import org.springframework.boot.buildpack.platform.json.AbstractJsonTests;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * Tests for {@link ManifestList}.
+ *
+ * @author Phillip Webb
+ */
+class ManifestListTests extends AbstractJsonTests {
+
+	@Test
+	void loadJsonFromDistributionManifestList() throws IOException {
+		String content = getContentAsString("distribution-manifest-list.json");
+		ManifestList manifestList = getManifestList(content);
+		assertThat(manifestList.getSchemaVersion()).isEqualTo(2);
+		assertThat(manifestList.getMediaType()).isEqualTo("application/vnd.docker.distribution.manifest.list.v2+json");
+		assertThat(manifestList.getManifests()).hasSize(2);
+	}
+
+	private ManifestList getManifestList(String content) throws IOException {
+		return new ManifestList(getObjectMapper().readTree(content));
+	}
+
+}

spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/test/java/org/springframework/boot/buildpack/platform/docker/type/ManifestTests.java

@@ -0,0 +1,56 @@
+/*
+ * Copyright 2012-2024 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.boot.buildpack.platform.docker.type;
+
+import java.io.IOException;
+
+import org.junit.jupiter.api.Test;
+
+import org.springframework.boot.buildpack.platform.json.AbstractJsonTests;
+
+import static org.assertj.core.api.Assertions.assertThat;
+
+/**
+ * Tests for {@link Manifest}.
+ *
+ * @author Phillip Webb
+ */
+class ManifestTests extends AbstractJsonTests {
+
+	@Test
+	void loadJsonFromDistributionManifest() throws IOException {
+		String content = getContentAsString("distribution-manifest.json");
+		Manifest manifestList = getManifest(content);
+		assertThat(manifestList.getSchemaVersion()).isEqualTo(2);
+		assertThat(manifestList.getMediaType()).isEqualTo("application/vnd.docker.distribution.manifest.v2+json");
+		assertThat(manifestList.getLayers()).hasSize(1);
+	}
+
+	@Test
+	void loadJsonFromImageManifest() throws IOException {
+		String content = getContentAsString("image-manifest.json");
+		Manifest manifestList = getManifest(content);
+		assertThat(manifestList.getSchemaVersion()).isEqualTo(2);
+		assertThat(manifestList.getMediaType()).isEqualTo("application/vnd.oci.image.manifest.v1+json");
+		assertThat(manifestList.getLayers()).hasSize(1);
+	}
+
+	private Manifest getManifest(String content) throws IOException {
+		return new Manifest(getObjectMapper().readTree(content));
+	}
+
+}

spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/test/resources/org/springframework/boot/buildpack/platform/docker/type/distribution-manifest-list.json

@@ -0,0 +1,24 @@
+{
+	"schemaVersion": 2,
+	"mediaType": "application/vnd.docker.distribution.manifest.list.v2+json",
+	"manifests": [
+		{
+			"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
+			"size": 428,
+			"digest": "sha256:6dba064234a3aa60f7da2e0f1f8b86dccb7df2841136f577b08bd6a89004cb23",
+			"platform": {
+				"architecture": "amd64",
+				"os": "linux"
+			}
+		},
+		{
+			"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
+			"size": 428,
+			"digest": "sha256:c036aba2c51a86a7a338f60af4730df725c2abff1b8b565d753896fd9533dfad",
+			"platform": {
+				"architecture": "arm64",
+				"os": "linux"
+			}
+		}
+	]
+}

spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/test/resources/org/springframework/boot/buildpack/platform/docker/type/distribution-manifest.json

@@ -0,0 +1,16 @@
+{
+	"schemaVersion": 2,
+	"mediaType": "application/vnd.docker.distribution.manifest.v2+json",
+	"config": {
+		"mediaType": "application/vnd.docker.container.image.v1+json",
+		"size": 1175,
+		"digest": "sha256:b2160a0f9037918d3ca2270fb90f656f425760b337a5ed3813c3a48c09825065"
+	},
+	"layers": [
+		{
+			"mediaType": "application/vnd.docker.image.rootfs.diff.tar.gzip",
+			"size": 4872935,
+			"digest": "sha256:13ac7da0441b95b1960de1b87ed2c1ef129026cc69b926ffbe734a7dcc4fa40c"
+		}
+	]
+}

spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/test/resources/org/springframework/boot/buildpack/platform/docker/type/image-archive-index.json

@@ -0,0 +1,15 @@
+{
+	"schemaVersion": 2,
+	"manifests": [
+		{
+			"mediaType": "application/vnd.docker.distribution.manifest.list.v2+json",
+			"digest": "sha256:3bbe02431d8e5124ffe816ec27bf6508b50edd1d10218be1a03e799a186b9004",
+			"size": 529,
+			"annotations": {
+				"containerd.io/distribution.source.gcr.io": "paketo-buildpacks/adoptium",
+				"io.containerd.image.name": "gcr.io/paketo-buildpacks/adoptium:latest",
+				"org.opencontainers.image.ref.name": "latest"
+			}
+		}
+	]
+}

spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/test/resources/org/springframework/boot/buildpack/platform/docker/type/image-manifest.json

@@ -0,0 +1,20 @@
+{
+	"schemaVersion": 2,
+	"mediaType": "application/vnd.oci.image.manifest.v1+json",
+	"config": {
+		"mediaType": "application/vnd.oci.image.config.v1+json",
+		"digest": "sha256:ee382dc5c080aa6af5ea716041eaa4442c9d461520388627dfe51709c679043e",
+		"size": 849,
+		"platform": {
+			"architecture": "amd64",
+			"os": "linux"
+		}
+	},
+	"layers": [
+		{
+			"mediaType": "application/vnd.oci.image.layer.v1.tar",
+			"digest": "sha256:5caae51697b248b905dca1a4160864b0e1a15c300981736555cdce6567e8d477",
+			"size": 6656
+		}
+	]
+}