Add pullPolicy option for image building

This commit adds a pullPolicy option to the configuration of the Maven
plugin spring-boot:build-image goal and the Gradle plugin bootBuildImage
task. The new option gives users control over pulling the builder image
and run image from a remote image registry to the local Docker daemon.

See gh-22736

spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/main/java/org/springframework/boot/buildpack/platform/build/AbstractBuildLog.java

@@ -42,22 +42,32 @@ public abstract class AbstractBuildLog implements BuildLog {
 
 	@Override
 	public Consumer<TotalProgressEvent> pullingBuilder(BuildRequest request, ImageReference imageReference) {
-		return getProgressConsumer(" > Pulling builder image '" + imageReference + "'");
+		return pullingImage(imageReference, ImageType.BUILDER);
 	}
 
 	@Override
 	public void pulledBuilder(BuildRequest request, Image image) {
-		log(" > Pulled builder image '" + getDigest(image) + "'");
+		pulledImage(image, ImageType.BUILDER);
 	}
 
 	@Override
 	public Consumer<TotalProgressEvent> pullingRunImage(BuildRequest request, ImageReference imageReference) {
-		return getProgressConsumer(" > Pulling run image '" + imageReference + "'");
+		return pullingImage(imageReference, ImageType.RUNNER);
 	}
 
 	@Override
 	public void pulledRunImage(BuildRequest request, Image image) {
-		log(" > Pulled run image '" + getDigest(image) + "'");
+		pulledImage(image, ImageType.RUNNER);
+	}
+
+	@Override
+	public Consumer<TotalProgressEvent> pullingImage(ImageReference imageReference, ImageType imageType) {
+		return getProgressConsumer(String.format(" > Pulling %s '%s'", imageType.getDescription(), imageReference));
+	}
+
+	@Override
+	public void pulledImage(Image image, ImageType imageType) {
+		log(String.format(" > Pulled %s '%s'", imageType.getDescription(), getDigest(image)));
 	}
 
 	@Override

spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/main/java/org/springframework/boot/buildpack/platform/build/BuildLog.java

@@ -71,6 +71,21 @@ public interface BuildLog {
 	 */
 	void pulledRunImage(BuildRequest request, Image image);
 
+	/**
+	 * Log that the image is being pulled.
+	 * @param imageReference the image reference
+	 * @param imageType the image type
+	 * @return a consumer for progress update events
+	 */
+	Consumer<TotalProgressEvent> pullingImage(ImageReference imageReference, ImageType imageType);
+
+	/**
+	 * Log that the image has been pulled.
+	 * @param image the builder image that was pulled
+	 * @param imageType the image type that was pulled
+	 */
+	void pulledImage(Image image, ImageType imageType);
+
 	/**
 	 * Log that the lifecycle is executing.
 	 * @param request the build request

spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/main/java/org/springframework/boot/buildpack/platform/build/BuildRequest.java

@@ -32,6 +32,7 @@ import org.springframework.util.Assert;
  *
  * @author Phillip Webb
  * @author Scott Frederick
+ * @author Andrey Shlykov
  * @since 2.3.0
  */
 public class BuildRequest {
@@ -56,6 +57,8 @@ public class BuildRequest {
 
 	private final boolean verboseLogging;
 
+	private final PullPolicy pullPolicy;
+
 	BuildRequest(ImageReference name, Function<Owner, TarArchive> applicationContent) {
 		Assert.notNull(name, "Name must not be null");
 		Assert.notNull(applicationContent, "ApplicationContent must not be null");
@@ -66,12 +69,13 @@ public class BuildRequest {
 		this.env = Collections.emptyMap();
 		this.cleanCache = false;
 		this.verboseLogging = false;
+		this.pullPolicy = PullPolicy.ALWAYS;
 		this.creator = Creator.withVersion("");
 	}
 
 	BuildRequest(ImageReference name, Function<Owner, TarArchive> applicationContent, ImageReference builder,
 			ImageReference runImage, Creator creator, Map<String, String> env, boolean cleanCache,
-			boolean verboseLogging) {
+			boolean verboseLogging, PullPolicy pullPolicy) {
 		this.name = name;
 		this.applicationContent = applicationContent;
 		this.builder = builder;
@@ -80,6 +84,7 @@ public class BuildRequest {
 		this.env = env;
 		this.cleanCache = cleanCache;
 		this.verboseLogging = verboseLogging;
+		this.pullPolicy = pullPolicy;
 	}
 
 	/**
@@ -90,7 +95,7 @@ public class BuildRequest {
 	public BuildRequest withBuilder(ImageReference builder) {
 		Assert.notNull(builder, "Builder must not be null");
 		return new BuildRequest(this.name, this.applicationContent, builder.inTaggedOrDigestForm(), this.runImage,
-				this.creator, this.env, this.cleanCache, this.verboseLogging);
+				this.creator, this.env, this.cleanCache, this.verboseLogging, this.pullPolicy);
 	}
 
 	/**
@@ -100,7 +105,7 @@ public class BuildRequest {
 	 */
 	public BuildRequest withRunImage(ImageReference runImageName) {
 		return new BuildRequest(this.name, this.applicationContent, this.builder, runImageName.inTaggedOrDigestForm(),
-				this.creator, this.env, this.cleanCache, this.verboseLogging);
+				this.creator, this.env, this.cleanCache, this.verboseLogging, this.pullPolicy);
 	}
 
 	/**
@@ -111,7 +116,7 @@ public class BuildRequest {
 	public BuildRequest withCreator(Creator creator) {
 		Assert.notNull(creator, "Creator must not be null");
 		return new BuildRequest(this.name, this.applicationContent, this.builder, this.runImage, creator, this.env,
-				this.cleanCache, this.verboseLogging);
+				this.cleanCache, this.verboseLogging, this.pullPolicy);
 	}
 
 	/**
@@ -126,7 +131,7 @@ public class BuildRequest {
 		Map<String, String> env = new LinkedHashMap<>(this.env);
 		env.put(name, value);
 		return new BuildRequest(this.name, this.applicationContent, this.builder, this.runImage, this.creator,
-				Collections.unmodifiableMap(env), this.cleanCache, this.verboseLogging);
+				Collections.unmodifiableMap(env), this.cleanCache, this.verboseLogging, this.pullPolicy);
 	}
 
 	/**
@@ -139,7 +144,7 @@ public class BuildRequest {
 		Map<String, String> updatedEnv = new LinkedHashMap<>(this.env);
 		updatedEnv.putAll(env);
 		return new BuildRequest(this.name, this.applicationContent, this.builder, this.runImage, this.creator,
-				Collections.unmodifiableMap(updatedEnv), this.cleanCache, this.verboseLogging);
+				Collections.unmodifiableMap(updatedEnv), this.cleanCache, this.verboseLogging, this.pullPolicy);
 	}
 
 	/**
@@ -149,7 +154,7 @@ public class BuildRequest {
 	 */
 	public BuildRequest withCleanCache(boolean cleanCache) {
 		return new BuildRequest(this.name, this.applicationContent, this.builder, this.runImage, this.creator, this.env,
-				cleanCache, this.verboseLogging);
+				cleanCache, this.verboseLogging, this.pullPolicy);
 	}
 
 	/**
@@ -159,7 +164,17 @@ public class BuildRequest {
 	 */
 	public BuildRequest withVerboseLogging(boolean verboseLogging) {
 		return new BuildRequest(this.name, this.applicationContent, this.builder, this.runImage, this.creator, this.env,
-				this.cleanCache, verboseLogging);
+				this.cleanCache, verboseLogging, this.pullPolicy);
+	}
+
+	/**
+	 * Return a new {@link BuildRequest} with the updated image pull policy.
+	 * @param pullPolicy image pull policy {@link PullPolicy}
+	 * @return an updated build request
+	 */
+	public BuildRequest withPullPolicy(PullPolicy pullPolicy) {
+		return new BuildRequest(this.name, this.applicationContent, this.builder, this.runImage, this.creator, this.env,
+				this.cleanCache, this.verboseLogging, pullPolicy);
 	}
 
 	/**
@@ -229,6 +244,14 @@ public class BuildRequest {
 		return this.verboseLogging;
 	}
 
+	/**
+	 * Return the image {@link PullPolicy} that the builder should use.
+	 * @return image pull policy
+	 */
+	public PullPolicy getPullPolicy() {
+		return this.pullPolicy;
+	}
+
 	/**
 	 * Factory method to create a new {@link BuildRequest} from a JAR file.
 	 * @param jarFile the source jar file

spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/main/java/org/springframework/boot/buildpack/platform/build/Builder.java

@@ -35,6 +35,7 @@ import org.springframework.util.StringUtils;
  *
  * @author Phillip Webb
  * @author Scott Frederick
+ * @author Andrey Shlykov
  * @since 2.3.0
  */
 public class Builder {
@@ -60,7 +61,7 @@ public class Builder {
 	public void build(BuildRequest request) throws DockerEngineException, IOException {
 		Assert.notNull(request, "Request must not be null");
 		this.log.start(request);
-		Image builderImage = pullBuilder(request);
+		Image builderImage = getImage(request, ImageType.BUILDER);
 		BuilderMetadata builderMetadata = BuilderMetadata.fromImage(builderImage);
 		BuildOwner buildOwner = BuildOwner.fromEnv(builderImage.getConfig().getEnv());
 		request = determineRunImage(request, builderImage, builderMetadata.getStack());
@@ -75,22 +76,13 @@ public class Builder {
 		}
 	}
 
-	private Image pullBuilder(BuildRequest request) throws IOException {
-		ImageReference builderImageReference = request.getBuilder();
-		Consumer<TotalProgressEvent> progressConsumer = this.log.pullingBuilder(request, builderImageReference);
-		TotalProgressPullListener listener = new TotalProgressPullListener(progressConsumer);
-		Image builderImage = this.docker.image().pull(builderImageReference, listener);
-		this.log.pulledBuilder(request, builderImage);
-		return builderImage;
-	}
-
 	private BuildRequest determineRunImage(BuildRequest request, Image builderImage, Stack builderStack)
 			throws IOException {
 		if (request.getRunImage() == null) {
 			ImageReference runImage = getRunImageReferenceForStack(builderStack);
 			request = request.withRunImage(runImage);
 		}
-		Image runImage = pullRunImage(request);
+		Image runImage = getImage(request, ImageType.RUNNER);
 		assertStackIdsMatch(runImage, builderImage);
 		return request;
 	}
@@ -101,12 +93,35 @@ public class Builder {
 		return ImageReference.of(name).inTaggedOrDigestForm();
 	}
 
-	private Image pullRunImage(BuildRequest request) throws IOException {
-		ImageReference runImage = request.getRunImage();
-		Consumer<TotalProgressEvent> progressConsumer = this.log.pullingRunImage(request, runImage);
+	private Image getImage(BuildRequest request, ImageType imageType) throws IOException {
+		ImageReference imageReference = (imageType == ImageType.BUILDER) ? request.getBuilder() : request.getRunImage();
+
+		Image image;
+		if (request.getPullPolicy() != PullPolicy.ALWAYS) {
+			try {
+				image = this.docker.image().inspect(imageReference);
+			}
+			catch (DockerEngineException exception) {
+				if (request.getPullPolicy() == PullPolicy.IF_NOT_PRESENT && exception.getStatusCode() == 404) {
+					image = pullImage(imageReference, imageType);
+				}
+				else {
+					throw exception;
+				}
+			}
+		}
+		else {
+			image = pullImage(imageReference, imageType);
+		}
+
+		return image;
+	}
+
+	private Image pullImage(ImageReference reference, ImageType imageType) throws IOException {
+		Consumer<TotalProgressEvent> progressConsumer = this.log.pullingImage(reference, imageType);
 		TotalProgressPullListener listener = new TotalProgressPullListener(progressConsumer);
-		Image image = this.docker.image().pull(runImage, listener);
-		this.log.pulledRunImage(request, image);
+		Image image = this.docker.image().pull(reference, listener);
+		this.log.pulledImage(image, imageType);
 		return image;
 	}
 

spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/main/java/org/springframework/boot/buildpack/platform/build/ImageType.java

@@ -0,0 +1,47 @@
+/*
+ * Copyright 2012-2020 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.boot.buildpack.platform.build;
+
+/**
+ * Image types.
+ *
+ * @author Andrey Shlykov
+ * @since 2.4.0
+ */
+public enum ImageType {
+
+	/**
+	 * Builder image.
+	 */
+	BUILDER("builder image"),
+
+	/**
+	 * Run image.
+	 */
+	RUNNER("run image");
+
+	private final String description;
+
+	ImageType(String description) {
+		this.description = description;
+	}
+
+	public String getDescription() {
+		return this.description;
+	}
+
+}

spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/main/java/org/springframework/boot/buildpack/platform/build/PullPolicy.java

@@ -0,0 +1,42 @@
+/*
+ * Copyright 2012-2020 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.boot.buildpack.platform.build;
+
+/**
+ * Image pull policy.
+ *
+ * @author Andrey Shlykov
+ * @since 2.4.0
+ */
+public enum PullPolicy {
+
+	/**
+	 * Always pull the image.
+	 */
+	ALWAYS,
+
+	/**
+	 * Never pull the image.
+	 */
+	NEVER,
+
+	/**
+	 * Pull the image if it does not already exist in registry.
+	 */
+	IF_NOT_PRESENT
+
+}

spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/main/java/org/springframework/boot/buildpack/platform/docker/DockerApi.java

@@ -158,10 +158,7 @@ public class DockerApi {
 						listener.onUpdate(event);
 					});
 				}
-				URI imageUri = buildUrl("/images/" + reference.withDigest(digestCapture.getCapturedDigest()) + "/json");
-				try (Response response = http().get(imageUri)) {
-					return Image.of(response.getContent());
-				}
+				return inspect(reference.withDigest(digestCapture.getCapturedDigest()));
 			}
 			finally {
 				listener.onFinish();
@@ -202,6 +199,20 @@ public class DockerApi {
 			http().delete(uri);
 		}
 
+		/**
+		 * Inspect an image.
+		 * @param reference the image reference
+		 * @return the image from the local repository
+		 * @throws IOException on IO error
+		 */
+		public Image inspect(ImageReference reference) throws IOException {
+			Assert.notNull(reference, "Reference must not be null");
+			URI imageUri = buildUrl("/images/" + reference + "/json");
+			try (Response response = http().get(imageUri)) {
+				return Image.of(response.getContent());
+			}
+		}
+
 	}
 
 	/**

spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/main/java/org/springframework/boot/buildpack/platform/docker/transport/DockerEngineException.java

@@ -38,7 +38,7 @@ public class DockerEngineException extends RuntimeException {
 
 	private final Message responseMessage;
 
-	DockerEngineException(String host, URI uri, int statusCode, String reasonPhrase, Errors errors,
+	public DockerEngineException(String host, URI uri, int statusCode, String reasonPhrase, Errors errors,
 			Message responseMessage) {
 		super(buildMessage(host, uri, statusCode, reasonPhrase, errors, responseMessage));
 		this.statusCode = statusCode;

spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/test/java/org/springframework/boot/buildpack/platform/build/BuilderTests.java

@@ -19,6 +19,7 @@ package org.springframework.boot.buildpack.platform.build;
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.PrintStream;
+import java.net.URI;
 
 import org.junit.jupiter.api.Test;
 import org.mockito.ArgumentCaptor;
@@ -29,6 +30,7 @@ import org.springframework.boot.buildpack.platform.docker.DockerApi.ContainerApi
 import org.springframework.boot.buildpack.platform.docker.DockerApi.ImageApi;
 import org.springframework.boot.buildpack.platform.docker.DockerApi.VolumeApi;
 import org.springframework.boot.buildpack.platform.docker.TotalProgressPullListener;
+import org.springframework.boot.buildpack.platform.docker.transport.DockerEngineException;
 import org.springframework.boot.buildpack.platform.docker.type.ContainerReference;
 import org.springframework.boot.buildpack.platform.docker.type.ContainerStatus;
 import org.springframework.boot.buildpack.platform.docker.type.Image;
@@ -44,6 +46,8 @@ import static org.mockito.ArgumentMatchers.any;
 import static org.mockito.ArgumentMatchers.eq;
 import static org.mockito.BDDMockito.given;
 import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
 import static org.mockito.Mockito.verify;
 
 /**
@@ -147,6 +151,86 @@ class BuilderTests {
 		verify(docker.image()).remove(archive.getValue().getTag(), true);
 	}
 
+	@Test
+	void buildInvokesBuilderWithNeverPullPolicy() throws Exception {
+		TestPrintStream out = new TestPrintStream();
+		DockerApi docker = mockDockerApi();
+		Image builderImage = loadImage("image.json");
+		Image runImage = loadImage("run-image.json");
+		given(docker.image().pull(eq(ImageReference.of(BuildRequest.DEFAULT_BUILDER_IMAGE_NAME)), any()))
+				.willAnswer(withPulledImage(builderImage));
+		given(docker.image().pull(eq(ImageReference.of("docker.io/cloudfoundry/run:base-cnb")), any()))
+				.willAnswer(withPulledImage(runImage));
+		given(docker.image().inspect(eq(ImageReference.of(BuildRequest.DEFAULT_BUILDER_IMAGE_NAME))))
+				.willReturn(builderImage);
+		given(docker.image().inspect(eq(ImageReference.of("docker.io/cloudfoundry/run:base-cnb"))))
+				.willReturn(runImage);
+		Builder builder = new Builder(BuildLog.to(out), docker);
+		BuildRequest request = getTestRequest().withPullPolicy(PullPolicy.NEVER);
+		builder.build(request);
+		assertThat(out.toString()).contains("Running creator");
+		assertThat(out.toString()).contains("Successfully built image 'docker.io/library/my-application:latest'");
+		ArgumentCaptor<ImageArchive> archive = ArgumentCaptor.forClass(ImageArchive.class);
+		verify(docker.image()).load(archive.capture(), any());
+		verify(docker.image()).remove(archive.getValue().getTag(), true);
+		verify(docker.image(), never()).pull(any(), any());
+		verify(docker.image(), times(2)).inspect(any());
+	}
+
+	@Test
+	void buildInvokesBuilderWithAlwaysPullPolicy() throws Exception {
+		TestPrintStream out = new TestPrintStream();
+		DockerApi docker = mockDockerApi();
+		Image builderImage = loadImage("image.json");
+		Image runImage = loadImage("run-image.json");
+		given(docker.image().pull(eq(ImageReference.of(BuildRequest.DEFAULT_BUILDER_IMAGE_NAME)), any()))
+				.willAnswer(withPulledImage(builderImage));
+		given(docker.image().pull(eq(ImageReference.of("docker.io/cloudfoundry/run:base-cnb")), any()))
+				.willAnswer(withPulledImage(runImage));
+		given(docker.image().inspect(eq(ImageReference.of(BuildRequest.DEFAULT_BUILDER_IMAGE_NAME))))
+				.willReturn(builderImage);
+		given(docker.image().inspect(eq(ImageReference.of("docker.io/cloudfoundry/run:base-cnb"))))
+				.willReturn(runImage);
+		Builder builder = new Builder(BuildLog.to(out), docker);
+		BuildRequest request = getTestRequest().withPullPolicy(PullPolicy.ALWAYS);
+		builder.build(request);
+		assertThat(out.toString()).contains("Running creator");
+		assertThat(out.toString()).contains("Successfully built image 'docker.io/library/my-application:latest'");
+		ArgumentCaptor<ImageArchive> archive = ArgumentCaptor.forClass(ImageArchive.class);
+		verify(docker.image()).load(archive.capture(), any());
+		verify(docker.image()).remove(archive.getValue().getTag(), true);
+		verify(docker.image(), times(2)).pull(any(), any());
+		verify(docker.image(), never()).inspect(any());
+	}
+
+	@Test
+	void buildInvokesBuilderWithIfNotPresentPullPolicy() throws Exception {
+		TestPrintStream out = new TestPrintStream();
+		DockerApi docker = mockDockerApi();
+		Image builderImage = loadImage("image.json");
+		Image runImage = loadImage("run-image.json");
+		given(docker.image().pull(eq(ImageReference.of(BuildRequest.DEFAULT_BUILDER_IMAGE_NAME)), any()))
+				.willAnswer(withPulledImage(builderImage));
+		given(docker.image().pull(eq(ImageReference.of("docker.io/cloudfoundry/run:base-cnb")), any()))
+				.willAnswer(withPulledImage(runImage));
+		given(docker.image().inspect(eq(ImageReference.of(BuildRequest.DEFAULT_BUILDER_IMAGE_NAME)))).willThrow(
+				new DockerEngineException("docker://localhost/", new URI("example"), 404, "NOT FOUND", null, null))
+				.willReturn(builderImage);
+		given(docker.image().inspect(eq(ImageReference.of("docker.io/cloudfoundry/run:base-cnb")))).willThrow(
+				new DockerEngineException("docker://localhost/", new URI("example"), 404, "NOT FOUND", null, null))
+				.willReturn(runImage);
+		Builder builder = new Builder(BuildLog.to(out), docker);
+		BuildRequest request = getTestRequest().withPullPolicy(PullPolicy.IF_NOT_PRESENT);
+		builder.build(request);
+		assertThat(out.toString()).contains("Running creator");
+		assertThat(out.toString()).contains("Successfully built image 'docker.io/library/my-application:latest'");
+		ArgumentCaptor<ImageArchive> archive = ArgumentCaptor.forClass(ImageArchive.class);
+		verify(docker.image()).load(archive.capture(), any());
+		verify(docker.image()).remove(archive.getValue().getTag(), true);
+		verify(docker.image(), times(2)).inspect(any());
+		verify(docker.image(), times(2)).pull(any(), any());
+	}
+
 	@Test
 	void buildWhenStackIdDoesNotMatchThrowsException() throws Exception {
 		TestPrintStream out = new TestPrintStream();

spring-boot-project/spring-boot-tools/spring-boot-buildpack-platform/src/test/java/org/springframework/boot/buildpack/platform/docker/DockerApiTests.java

@@ -217,6 +217,21 @@ class DockerApiTests {
 			verify(http()).delete(removeUri);
 		}
 
+		@Test
+		void inspectWhenReferenceIsNullThrowsException() {
+			assertThatIllegalArgumentException().isThrownBy(() -> this.api.inspect(null))
+					.withMessage("Reference must not be null");
+		}
+
+		@Test
+		void inspectInspectImage() throws Exception {
+			ImageReference reference = ImageReference.of("gcr.io/paketo-buildpacks/builder:base");
+			URI imageUri = new URI(IMAGES_URL + "/gcr.io/paketo-buildpacks/builder:base/json");
+			given(http().get(imageUri)).willReturn(responseOf("type/image.json"));
+			Image image = this.api.inspect(reference);
+			assertThat(image.getLayers()).hasSize(46);
+		}
+
 	}
 
 	@Nested