Adding zero downtime backing app updates

Finishes #113

build.gradle

@@ -48,7 +48,7 @@ configure(allprojects) {
 		reactorVersion = project.findProperty("reactorVersion") ?: "Californium-SR3"
 		openServiceBrokerVersion = "3.0.0.M3"
 		springCredhubVersion = "2.0.0.BUILD-SNAPSHOT"
-		cfJavaClientVersion = "3.14.0.RELEASE"
+		cfJavaClientVersion = "3.15.0.RELEASE"
 		mockitoVersion = "2.23.4"
 		immutablesVersion = "2.7.3"
 		assertjVersion = "3.9.1"

spring-cloud-app-broker-acceptance-tests/src/test/java/org.springframework.cloud.appbroker.acceptance/CloudFoundryAcceptanceTest.java

@@ -51,13 +51,17 @@ import org.springframework.boot.test.context.SpringBootTest;
 import org.springframework.cloud.appbroker.acceptance.fixtures.cf.CloudFoundryClientConfiguration;
 import org.springframework.cloud.appbroker.acceptance.fixtures.cf.CloudFoundryService;
 import org.springframework.test.context.junit.jupiter.SpringExtension;
+import org.springframework.web.client.RestTemplate;
 
 import static org.assertj.core.api.Assertions.assertThat;
 
 @SpringBootTest(classes = {
 	CloudFoundryClientConfiguration.class,
 	CloudFoundryService.class,
-	UaaService.class})
+	UaaService.class,
+	HealthListener.class,
+	RestTemplate.class
+})
 @ExtendWith(SpringExtension.class)
 @ExtendWith(BrokerPropertiesParameterResolver.class)
 @EnableConfigurationProperties(AcceptanceTestProperties.class)

spring-cloud-app-broker-acceptance-tests/src/test/java/org.springframework.cloud.appbroker.acceptance/HealthListener.java

@@ -0,0 +1,85 @@
+/*
+ * Copyright 2016-2018. the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.cloud.appbroker.acceptance;
+
+import java.net.URI;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.springframework.http.ResponseEntity;
+import org.springframework.stereotype.Service;
+import org.springframework.web.client.RestClientException;
+import org.springframework.web.client.RestTemplate;
+
+@Service
+public class HealthListener {
+
+	private final AtomicInteger requests = new AtomicInteger();
+	private final AtomicInteger errors = new AtomicInteger();
+	private final AtomicBoolean running = new AtomicBoolean(false);
+	private Thread runner;
+	private final RestTemplate restTemplate;
+
+	public HealthListener(RestTemplate restTemplate) {
+		this.restTemplate = restTemplate;
+	}
+
+	void start(String path) {
+		if (running.get()) {
+			throw new IllegalStateException("cannot start when test is already running");
+		}
+		requests.set(0);
+		errors.set(0);
+		running.set(true);
+
+		runner = new Thread(() -> {
+			while (running.get()) {
+				try {
+					requests.incrementAndGet();
+					ResponseEntity<String> response = restTemplate.getForEntity(URI.create("http://" + path + "/actuator/health"), String.class);
+					if (response.getStatusCodeValue() != 200) {
+						errors.incrementAndGet();
+					}
+					Thread.sleep(1000);
+				}
+				catch (RestClientException | InterruptedException re) {
+					errors.incrementAndGet();
+				}
+			}
+		});
+		runner.start();
+	}
+
+	void stop() {
+		running.set(false);
+		try {
+			runner.join();
+		}
+		catch (InterruptedException e) {
+			e.printStackTrace();
+		}
+	}
+
+	int getSuccesses() {
+		return requests.get();
+	}
+
+	int getFailures() {
+		return errors.get();
+	}
+
+}

spring-cloud-app-broker-acceptance-tests/src/test/java/org.springframework.cloud.appbroker.acceptance/UpdateInstanceAcceptanceTest.java

@@ -24,6 +24,8 @@ import com.jayway.jsonpath.DocumentContext;
 import org.cloudfoundry.operations.applications.ApplicationSummary;
 import org.junit.jupiter.api.Test;
 
+import org.springframework.beans.factory.annotation.Autowired;
+
 import static com.revinate.assertj.json.JsonPathAssert.assertThat;
 import static org.assertj.core.api.Assertions.assertThat;
 
@@ -32,6 +34,9 @@ class UpdateInstanceAcceptanceTest extends CloudFoundryAcceptanceTest {
 	private static final String APP_NAME = "app-update";
 	private static final String SI_NAME = "si-update";
 
+	@Autowired
+	private HealthListener healthListener;
+
 	@Test
 	@AppBrokerTestProperties({
 		"spring.cloud.appbroker.services[0].service-name=" + APP_SERVICE_NAME,
@@ -60,6 +65,9 @@ class UpdateInstanceAcceptanceTest extends CloudFoundryAcceptanceTest {
 		assertThat(json).jsonPathAsString("$.parameter2").isEqualTo("config2");
 		assertThat(json).jsonPathAsString("$.parameter3").isEqualTo("config3");
 
+		String path = backingApplication.get().getUrls().get(0);
+		healthListener.start(path);
+
 		// when the service instance is updated
 		Map<String, Object> parameters = new HashMap<>();
 		parameters.put("parameter1", "value1");
@@ -67,6 +75,11 @@ class UpdateInstanceAcceptanceTest extends CloudFoundryAcceptanceTest {
 		parameters.put("parameter3", "value3");
 		updateServiceInstance(SI_NAME, parameters);
 
+		// then the backing application was updated with zero downtime
+		healthListener.stop();
+		assertThat(healthListener.getFailures()).isEqualTo(0);
+		assertThat(healthListener.getSuccesses()).isGreaterThan(0);
+
 		// the backing application is updated with the new parameters
 		json = getSpringAppJson(APP_NAME);
 		assertThat(json).jsonPathAsString("$.parameter1").isEqualTo("value1");

spring-cloud-app-broker-acceptance-tests/src/test/java/org.springframework.cloud.appbroker.acceptance/UpdateInstanceWithServicesAcceptanceTest.java

@@ -24,6 +24,8 @@ import java.util.HashMap;
 import java.util.Map;
 import java.util.Optional;
 
+import org.springframework.beans.factory.annotation.Autowired;
+
 import static org.assertj.core.api.Assertions.assertThat;
 
 class UpdateInstanceWithServicesAcceptanceTest extends CloudFoundryAcceptanceTest {
@@ -33,6 +35,9 @@ class UpdateInstanceWithServicesAcceptanceTest extends CloudFoundryAcceptanceTes
 
 	private static final String BACKING_SI_NAME = "backing-service-instance-update";
 
+	@Autowired
+	private HealthListener healthListener;
+
 	@Test
 	@AppBrokerTestProperties({
 		"spring.cloud.appbroker.services[0].service-name=" + APP_SERVICE_NAME,
@@ -60,6 +65,9 @@ class UpdateInstanceWithServicesAcceptanceTest extends CloudFoundryAcceptanceTes
 		ServiceInstance backingServiceInstance = getServiceInstance(BACKING_SI_NAME);
 		assertThat(backingServiceInstance.getApplications()).contains(APP_NAME);
 
+		String path = backingApplication.get().getUrls().get(0);
+		healthListener.start(path);
+
 		// when the service instance is updated
 		Map<String, Object> parameters = new HashMap<>();
 		parameters.put("parameter1", "value1");
@@ -67,6 +75,11 @@ class UpdateInstanceWithServicesAcceptanceTest extends CloudFoundryAcceptanceTes
 		parameters.put("parameter3", "value3");
 		updateServiceInstance(SI_NAME, parameters);
 
+		// then the backing application was updated with zero downtime
+		healthListener.stop();
+		assertThat(healthListener.getFailures()).isEqualTo(0);
+		assertThat(healthListener.getSuccesses()).isGreaterThan(0);
+
 		// then a backing application is re-deployed
 		Optional<ApplicationSummary> updatedBackingApplication = getApplicationSummary(APP_NAME);
 		assertThat(updatedBackingApplication).hasValueSatisfying(app ->

spring-cloud-app-broker-acceptance-tests/src/test/java/org.springframework.cloud.appbroker.acceptance/UpdateInstanceWithTargetAcceptanceTest.java

@@ -24,6 +24,8 @@ import com.jayway.jsonpath.DocumentContext;
 import org.cloudfoundry.operations.applications.ApplicationSummary;
 import org.junit.jupiter.api.Test;
 
+import org.springframework.beans.factory.annotation.Autowired;
+
 import static com.revinate.assertj.json.JsonPathAssert.assertThat;
 import static org.assertj.core.api.Assertions.assertThat;
 
@@ -32,6 +34,9 @@ class UpdateInstanceWithTargetAcceptanceTest extends CloudFoundryAcceptanceTest
 	private static final String APP_NAME = "app-update-target";
 	private static final String SI_NAME = "si-update-target";
 
+	@Autowired
+	private HealthListener healthListener;
+
 	@Test
 	@AppBrokerTestProperties({
 		"spring.cloud.appbroker.services[0].service-name=" + APP_SERVICE_NAME,
@@ -59,9 +64,17 @@ class UpdateInstanceWithTargetAcceptanceTest extends CloudFoundryAcceptanceTest
 			assertThat(app.getUrls().get(0)).startsWith(APP_NAME + "-" + spaceName);
 		});
 
+		String path = backingApplication.get().getUrls().get(0);
+		healthListener.start(path);
+
 		// when the service instance is updated
 		updateServiceInstance(SI_NAME, Collections.singletonMap("parameter2", "config2"));
 
+		// then the backing application was updated with zero downtime
+		healthListener.stop();
+		assertThat(healthListener.getFailures()).isEqualTo(0);
+		assertThat(healthListener.getSuccesses()).isGreaterThan(0);
+
 		// then the service instance has the initial parameters
 		DocumentContext json = getSpringAppJson(APP_NAME, spaceName);
 		assertThat(json).jsonPathAsString("$.parameter1").isEqualTo("config1");

spring-cloud-app-broker-core/src/main/java/org/springframework/cloud/appbroker/deployer/BackingAppDeploymentService.java

@@ -45,6 +45,18 @@ public class BackingAppDeploymentService {
 			.sequential();
 	}
 
+	public Flux<String> update(List<BackingApplication> backingApps) {
+		return Flux.fromIterable(backingApps)
+			.parallel()
+			.runOn(Schedulers.parallel())
+			.flatMap(deployerClient::update)
+			.doOnRequest(l -> log.debug("Updating applications {}", backingApps))
+			.doOnEach(response -> log.debug("Finished updating application {}", response))
+			.doOnComplete(() -> log.debug("Finished updating application {}", backingApps))
+			.doOnError(exception -> log.error("Error updating applications {} with error {}", backingApps, exception))
+			.sequential();
+	}
+
 	public Flux<String> undeploy(List<BackingApplication> backingApps) {
 		return Flux.fromIterable(backingApps)
 			.parallel()

spring-cloud-app-broker-core/src/main/java/org/springframework/cloud/appbroker/deployer/DeployerClient.java

@@ -47,6 +47,20 @@ public class DeployerClient {
 			.map(DeployApplicationResponse::getName);
 	}
 
+	Mono<String> update(BackingApplication backingApplication) {
+		return appDeployer.update(UpdateApplicationRequest.builder()
+			.name(backingApplication.getName())
+			.path(backingApplication.getPath())
+			.properties(backingApplication.getProperties())
+			.environment(backingApplication.getEnvironment())
+			.services(backingApplication.getServices().stream().map(ServicesSpec::getServiceInstanceName).collect(Collectors.toList()))
+			.build())
+			.doOnRequest(l -> log.debug("Deploying application {}", backingApplication))
+			.doOnSuccess(response -> log.debug("Finished updating application {}", backingApplication))
+			.doOnError(exception -> log.error("Error updating application {} with error {}", backingApplication, exception))
+			.map(UpdateApplicationResponse::getName);
+	}
+
 	Mono<String> undeploy(BackingApplication backingApplication) {
 		return appDeployer.undeploy(UndeployApplicationRequest.builder()
 			.properties(backingApplication.getProperties())

spring-cloud-app-broker-core/src/main/java/org/springframework/cloud/appbroker/workflow/instance/AppDeploymentUpdateServiceInstanceWorkflow.java

@@ -78,7 +78,7 @@ public class AppDeploymentUpdateServiceInstanceWorkflow
 								request.getServiceInstanceId()))
 						.flatMap(backingApps ->
 							appsParametersTransformationService.transformParameters(backingApps, request.getParameters()))
-						.flatMapMany(deploymentService::deploy)
+						.flatMapMany(deploymentService::update)
 						.doOnRequest(l -> log.debug("Deploying applications {}", brokeredServices))
 						.doOnEach(result -> log.debug("Finished deploying {}", result))
 						.doOnComplete(() -> log.debug("Finished deploying applications {}", brokeredServices))

spring-cloud-app-broker-core/src/test/java/org/springframework/cloud/appbroker/workflow/instance/AppDeploymentUpdateServiceInstanceWorkflowTest.java

@@ -136,7 +136,7 @@ class AppDeploymentUpdateServiceInstanceWorkflowTest {
 			.expectNext()
 			.verifyComplete();
 
-		verify(appDeploymentService).deploy(backingApps);
+		verify(appDeploymentService).update(backingApps);
 		verify(servicesProvisionService).updateServiceInstance(backingServices);
 
 		final String expectedServiceId = "service-instance-id";
@@ -176,7 +176,7 @@ class AppDeploymentUpdateServiceInstanceWorkflowTest {
 	}
 
 	private void setupMocks(UpdateServiceInstanceRequest request) {
-		given(this.appDeploymentService.deploy(eq(backingApps)))
+		given(this.appDeploymentService.update(eq(backingApps)))
 			.willReturn(Flux.just("app1", "app2"));
 		given(this.servicesProvisionService.updateServiceInstance(eq(backingServices)))
 			.willReturn(Flux.just("my-service-instance"));

spring-cloud-app-broker-deployer-cloudfoundry/src/main/java/org/springframework/cloud/appbroker/deployer/cloudfoundry/CloudFoundryAppDeployer.java

@@ -18,6 +18,7 @@ package org.springframework.cloud.appbroker.deployer.cloudfoundry;
 
 import java.io.IOException;
 import java.nio.file.Path;
+import java.nio.file.Paths;
 import java.time.Duration;
 import java.util.HashMap;
 import java.util.HashSet;
@@ -25,6 +26,7 @@ import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
 import java.util.function.Consumer;
+import java.util.function.Function;
 import java.util.function.Predicate;
 import java.util.stream.Collectors;
 
@@ -35,12 +37,37 @@ import org.cloudfoundry.UnknownCloudFoundryException;
 import org.cloudfoundry.client.CloudFoundryClient;
 import org.cloudfoundry.client.v2.spaces.CreateSpaceRequest;
 import org.cloudfoundry.client.v2.spaces.DeleteSpaceRequest;
+import org.cloudfoundry.client.v3.Relationship;
+import org.cloudfoundry.client.v3.ToOneRelationship;
+import org.cloudfoundry.client.v3.builds.BuildState;
+import org.cloudfoundry.client.v3.builds.CreateBuildRequest;
+import org.cloudfoundry.client.v3.builds.CreateBuildResponse;
+import org.cloudfoundry.client.v3.builds.GetBuildRequest;
+import org.cloudfoundry.client.v3.builds.GetBuildResponse;
+import org.cloudfoundry.client.v3.deployments.CreateDeploymentRequest;
+import org.cloudfoundry.client.v3.deployments.CreateDeploymentResponse;
+import org.cloudfoundry.client.v3.deployments.DeploymentRelationships;
+import org.cloudfoundry.client.v3.deployments.DeploymentState;
+import org.cloudfoundry.client.v3.deployments.GetDeploymentRequest;
+import org.cloudfoundry.client.v3.deployments.GetDeploymentResponse;
+import org.cloudfoundry.client.v3.packages.CreatePackageRequest;
+import org.cloudfoundry.client.v3.packages.CreatePackageResponse;
+import org.cloudfoundry.client.v3.packages.GetPackageRequest;
+import org.cloudfoundry.client.v3.packages.GetPackageResponse;
+import org.cloudfoundry.client.v3.packages.Package;
+import org.cloudfoundry.client.v3.packages.PackageRelationships;
+import org.cloudfoundry.client.v3.packages.PackageState;
+import org.cloudfoundry.client.v3.packages.PackageType;
+import org.cloudfoundry.client.v3.packages.UploadPackageRequest;
+import org.cloudfoundry.client.v3.packages.UploadPackageResponse;
 import org.cloudfoundry.operations.CloudFoundryOperations;
 import org.cloudfoundry.operations.DefaultCloudFoundryOperations;
+import org.cloudfoundry.operations.applications.ApplicationDetail;
 import org.cloudfoundry.operations.applications.ApplicationHealthCheck;
 import org.cloudfoundry.operations.applications.ApplicationManifest;
 import org.cloudfoundry.operations.applications.DeleteApplicationRequest;
 import org.cloudfoundry.operations.applications.Docker;
+import org.cloudfoundry.operations.applications.GetApplicationRequest;
 import org.cloudfoundry.operations.applications.PushApplicationManifestRequest;
 import org.cloudfoundry.operations.applications.Route;
 import org.cloudfoundry.operations.organizations.OrganizationDetail;
@@ -50,6 +77,8 @@ import org.cloudfoundry.operations.services.ServiceInstance;
 import org.cloudfoundry.operations.services.UnbindServiceInstanceRequest;
 import org.cloudfoundry.operations.spaces.GetSpaceRequest;
 import org.cloudfoundry.operations.spaces.SpaceDetail;
+import org.cloudfoundry.util.DelayUtils;
+import org.reactivestreams.Publisher;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import reactor.core.Exceptions;
@@ -66,6 +95,8 @@ import org.springframework.cloud.appbroker.deployer.DeployApplicationResponse;
 import org.springframework.cloud.appbroker.deployer.DeploymentProperties;
 import org.springframework.cloud.appbroker.deployer.UndeployApplicationRequest;
 import org.springframework.cloud.appbroker.deployer.UndeployApplicationResponse;
+import org.springframework.cloud.appbroker.deployer.UpdateApplicationRequest;
+import org.springframework.cloud.appbroker.deployer.UpdateApplicationResponse;
 import org.springframework.cloud.appbroker.deployer.UpdateServiceInstanceRequest;
 import org.springframework.cloud.appbroker.deployer.UpdateServiceInstanceResponse;
 import org.springframework.cloud.appbroker.deployer.util.ByteSizeUtils;
@@ -110,7 +141,7 @@ public class CloudFoundryAppDeployer implements AppDeployer, ResourceLoaderAware
 	@Override
 	public Mono<DeployApplicationResponse> deploy(DeployApplicationRequest request) {
 		String appName = request.getName();
-		Resource appResource = getAppResource(request);
+		Resource appResource = getAppResource(request.getPath());
 		Map<String, String> deploymentProperties = request.getProperties();
 
 		logger.trace("Deploying application: request={}, resource={}",
@@ -132,6 +163,150 @@ public class CloudFoundryAppDeployer implements AppDeployer, ResourceLoaderAware
 					.build());
 	}
 
+	@Override
+	public Mono<UpdateApplicationResponse> update(UpdateApplicationRequest request) {
+		final String name = request.getName();
+		final Map<String, Object> environmentVariables =
+			getApplicationEnvironment(request.getProperties(), request.getEnvironment());
+
+		return this.operations
+			.applications()
+			.get(GetApplicationRequest.builder().name(name).build())
+			.map(ApplicationDetail::getId)
+			.flatMap(applicationId ->
+				updateApplicationEnvironment(environmentVariables, applicationId)
+					.thenReturn(applicationId)
+			)
+			.flatMap(applicationId -> Mono.zip(Mono.just(applicationId),
+				createPackageForApplication(applicationId)))
+			.map(tuple2 -> tuple2.mapT2(CreatePackageResponse::getId))
+			.flatMap(tuple2 -> {
+				String packageId = tuple2.getT2();
+				return Mono.zip(Mono.just(tuple2.getT1()), uploadPackage(request, packageId));
+			})
+
+			.map(tuple2 -> tuple2.mapT2(Package::getId))
+			.flatMap(tuple2 -> {
+					String packageId1 = tuple2.getT2();
+					return Mono.zip(Mono.just(tuple2.getT1()), waitForPackageReady(packageId1));
+				}
+			)
+			.map(tuple2 -> tuple2.mapT2(Package::getId))
+			.flatMap(tuple2 -> {
+				String packageId = tuple2.getT2();
+				return Mono.zip(Mono.just(tuple2.getT1()), createBuildForPackage(packageId));
+			})
+			.flatMap(tuple2 -> {
+					String buildId = tuple2.getT2();
+					return Mono.zip(Mono.just(tuple2.getT1()), waitForBuildStaged(buildId));
+				}
+			)
+			.map(tuple2 -> tuple2.mapT2((t2) -> t2.getDroplet().getId()))
+			.flatMap(tuple2 -> {
+				String dropletId = tuple2.getT2();
+				String applicationId = tuple2.getT1();
+				return createDeployment(dropletId, applicationId);
+			})
+			.map(CreateDeploymentResponse::getId)
+			.flatMap(this::waitForDeploymentDeployed)
+			.thenReturn(UpdateApplicationResponse.builder().name(name).build());
+	}
+
+	private Mono<GetDeploymentResponse> waitForDeploymentDeployed(String deploymentId) {
+		return this.client.deploymentsV3()
+						  .get(GetDeploymentRequest
+							  .builder()
+							  .deploymentId(deploymentId)
+							  .build())
+						  .filter(p -> p.getState().equals(DeploymentState.DEPLOYED))
+						  .repeatWhenEmpty(getExponentialBackOff());
+	}
+
+	private Function<Flux<Long>, Publisher<?>> getExponentialBackOff() {
+		return DelayUtils.exponentialBackOff(Duration.ofSeconds(2), Duration.ofSeconds(15), Duration.ofMinutes(10));
+	}
+
+	private Mono<CreateDeploymentResponse> createDeployment(String dropletId, String applicationId) {
+		return this.client.deploymentsV3()
+						  .create(CreateDeploymentRequest
+					   .builder()
+					   .droplet(Relationship
+						   .builder()
+						   .id(dropletId).build()).relationships(DeploymentRelationships
+						   .builder()
+						   .app(ToOneRelationship
+							   .builder()
+							   .data(Relationship.builder().id(applicationId).build())
+							   .build()
+						   ).build())
+					   .build());
+	}
+
+	private Mono<GetBuildResponse> waitForBuildStaged(String buildId) {
+		return this.client.builds().get(GetBuildRequest.builder().buildId(buildId).build())
+				   .filter(p -> p.getState().equals(BuildState.STAGED))
+				   .repeatWhenEmpty(getExponentialBackOff());
+	}
+
+	private Mono<String> createBuildForPackage(String packageId) {
+		return this.client.builds()
+						  .create(CreateBuildRequest
+							  .builder()
+							  .getPackage(Relationship.builder().id(packageId).build())
+							  .build())
+						  .map(CreateBuildResponse::getId);
+	}
+
+	private Mono<GetPackageResponse> waitForPackageReady(String packageId1) {
+		return this.client.packages()
+				   .get(GetPackageRequest.builder().packageId(packageId1).build())
+				   .filter(p -> p.getState().equals(PackageState.READY))
+				   .repeatWhenEmpty(getExponentialBackOff());
+	}
+
+	private Mono<UploadPackageResponse> uploadPackage(UpdateApplicationRequest request, String packageId) {
+		try {
+			return this.client.packages()
+							  .upload(UploadPackageRequest
+								  .builder()
+								  .packageId(packageId)
+								  .bits(Paths.get(getAppResource(request.getPath()).getURI()))
+								  .build());
+		}
+		catch (IOException e) {
+			throw Exceptions.propagate(e);
+		}
+	}
+
+	private Mono<CreatePackageResponse> createPackageForApplication(String applicationId) {
+		return this.client
+			.packages()
+			.create(CreatePackageRequest
+				.builder()
+				.relationships(PackageRelationships
+					.builder()
+					.application(ToOneRelationship
+						.builder()
+						.data(Relationship
+							.builder()
+							.id(applicationId)
+							.build())
+						.build())
+					.build())
+				.type(PackageType.BITS)
+				.build());
+	}
+
+	private Mono<org.cloudfoundry.client.v2.applications.UpdateApplicationResponse> updateApplicationEnvironment(
+		Map<String, Object> environmentVariables, String applicationId) {
+		return this.client.applicationsV2()
+						  .update(org.cloudfoundry.client.v2.applications.UpdateApplicationRequest
+							  .builder()
+							  .applicationId(applicationId)
+							  .putAllEnvironmentJsons(environmentVariables)
+							  .build());
+	}
+
 	private Mono<Void> pushApplication(DeployApplicationRequest request,
 									   Map<String, String> deploymentProperties,
 									   Resource appResource) {
@@ -468,8 +643,8 @@ public class CloudFoundryAppDeployer implements AppDeployer, ResourceLoaderAware
 		}
 	}
 
-	private Resource getAppResource(DeployApplicationRequest request) {
-		return resourceLoader.getResource(request.getPath());
+	private Resource getAppResource(String path) {
+		return resourceLoader.getResource(path);
 	}
 
 	/**

spring-cloud-app-broker-deployer-cloudfoundry/src/test/java/org/springframework/cloud/appbroker/deployer/cloudfoundry/CloudFoundryAppDeployerTest.java

@@ -117,7 +117,7 @@ class CloudFoundryAppDeployerTest {
 			.name(APP_NAME)
 			.path(APP_PATH)
 			.build();
-		
+
 		StepVerifier.create(appDeployer.deploy(request))
 			.assertNext(response -> assertThat(response.getName()).isEqualTo(APP_NAME))
 			.verifyComplete();

spring-cloud-app-broker-deployer-cloudfoundry/src/test/java/org/springframework/cloud/appbroker/deployer/cloudfoundry/CloudFoundryAppDeployerUpdateApplicationTest.java

@@ -0,0 +1,220 @@
+/*
+ * Copyright 2016-2018 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.cloud.appbroker.deployer.cloudfoundry;
+
+import org.cloudfoundry.client.CloudFoundryClient;
+import org.cloudfoundry.client.v2.applications.ApplicationsV2;
+import org.cloudfoundry.client.v2.applications.UpdateApplicationResponse;
+import org.cloudfoundry.client.v3.BuildpackData;
+import org.cloudfoundry.client.v3.Lifecycle;
+import org.cloudfoundry.client.v3.LifecycleType;
+import org.cloudfoundry.client.v3.Relationship;
+import org.cloudfoundry.client.v3.builds.BuildState;
+import org.cloudfoundry.client.v3.builds.Builds;
+import org.cloudfoundry.client.v3.builds.CreateBuildResponse;
+import org.cloudfoundry.client.v3.builds.CreatedBy;
+import org.cloudfoundry.client.v3.builds.Droplet;
+import org.cloudfoundry.client.v3.builds.GetBuildResponse;
+import org.cloudfoundry.client.v3.deployments.CreateDeploymentResponse;
+import org.cloudfoundry.client.v3.deployments.DeploymentState;
+import org.cloudfoundry.client.v3.deployments.DeploymentsV3;
+import org.cloudfoundry.client.v3.deployments.GetDeploymentResponse;
+import org.cloudfoundry.client.v3.packages.BitsData;
+import org.cloudfoundry.client.v3.packages.CreatePackageResponse;
+import org.cloudfoundry.client.v3.packages.GetPackageResponse;
+import org.cloudfoundry.client.v3.packages.PackageState;
+import org.cloudfoundry.client.v3.packages.PackageType;
+import org.cloudfoundry.client.v3.packages.Packages;
+import org.cloudfoundry.client.v3.packages.UploadPackageResponse;
+import org.cloudfoundry.operations.CloudFoundryOperations;
+import org.cloudfoundry.operations.applications.ApplicationDetail;
+import org.cloudfoundry.operations.applications.Applications;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+import org.mockito.junit.jupiter.MockitoSettings;
+import org.mockito.quality.Strictness;
+import reactor.core.publisher.Mono;
+import reactor.test.StepVerifier;
+
+import org.springframework.cloud.appbroker.deployer.AppDeployer;
+import org.springframework.cloud.appbroker.deployer.UpdateApplicationRequest;
+import org.springframework.core.io.FileSystemResource;
+import org.springframework.core.io.ResourceLoader;
+
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.when;
+
+@ExtendWith(MockitoExtension.class)
+@MockitoSettings(strictness = Strictness.LENIENT)
+class CloudFoundryAppDeployerUpdateApplicationTest {
+
+	private static final String APP_NAME = "test-app";
+	private static final String APP_PATH = "test.jar";
+
+	private AppDeployer appDeployer;
+
+	@Mock
+	private Applications operationsApplications;
+
+	@Mock
+	private ApplicationsV2 applicationsV2;
+
+	@Mock
+	private Builds builds;
+
+	@Mock
+	private DeploymentsV3 deploymentsV3;
+
+	@Mock
+	private Packages packages;
+
+	@Mock
+	private CloudFoundryOperations cloudFoundryOperations;
+
+	@Mock
+	private CloudFoundryClient cloudFoundryClient;
+
+	@Mock
+	private ResourceLoader resourceLoader;
+
+	@BeforeEach
+	void setUp() {
+		CloudFoundryDeploymentProperties deploymentProperties = new CloudFoundryDeploymentProperties();
+		CloudFoundryTargetProperties targetProperties = new CloudFoundryTargetProperties();
+
+		when(operationsApplications.pushManifest(any())).thenReturn(Mono.empty());
+		when(resourceLoader.getResource(APP_PATH)).thenReturn(new FileSystemResource(APP_PATH));
+
+		when(cloudFoundryOperations.applications()).thenReturn(operationsApplications);
+		when(cloudFoundryClient.applicationsV2()).thenReturn(applicationsV2);
+		when(cloudFoundryClient.packages()).thenReturn(packages);
+		when(cloudFoundryClient.builds()).thenReturn(builds);
+		when(cloudFoundryClient.deploymentsV3()).thenReturn(deploymentsV3);
+
+		appDeployer = new CloudFoundryAppDeployer(deploymentProperties,
+			cloudFoundryOperations, cloudFoundryClient, targetProperties, resourceLoader);
+	}
+
+	@Test
+	void updateApp() {
+		when(operationsApplications.get(any()))
+			.thenReturn(Mono.just(createApplicationDetail()));
+		when(applicationsV2.update(any()))
+			.thenReturn(Mono.just(UpdateApplicationResponse.builder().build()));
+		when(packages.create(any()))
+			.thenReturn(Mono.just(CreatePackageResponse
+				.builder()
+				.data(BitsData.builder().build())
+				.state(PackageState.READY)
+				.type(PackageType.BITS)
+				.createdAt("DATETIME")
+				.id("package-id")
+				.build()));
+
+		when(packages.upload(any()))
+			.thenReturn(Mono.just(UploadPackageResponse
+				.builder()
+				.data(BitsData.builder().build())
+				.state(PackageState.READY)
+				.type(PackageType.BITS)
+				.createdAt("DATETIME")
+				.id("package-id")
+				.build()));
+
+
+		when(packages.get(any()))
+			.thenReturn(Mono.just(GetPackageResponse
+				.builder()
+				.data(BitsData.builder().build())
+				.state(PackageState.READY)
+				.type(PackageType.BITS)
+				.createdAt("DATETIME")
+				.id("package-id")
+				.build()));
+
+		when(builds.create(any()))
+			.thenReturn(Mono.just(CreateBuildResponse
+				.builder()
+				.state(BuildState.STAGING)
+				.createdBy(CreatedBy.builder().id("create-by-id").email("an-email").name("creator").build())
+				.inputPackage(Relationship.builder().id("package-id").build())
+				.lifecycle(createLifecycle())
+				.createdAt("DATETIME")
+				.id("build-id")
+				.build()));
+		when(builds.get(any()))
+			.thenReturn(Mono.just(GetBuildResponse
+				.builder()
+				.state(BuildState.STAGED)
+				.createdBy(CreatedBy.builder().id("create-by-id").email("an-email").name("creator").build())
+				.inputPackage(Relationship.builder().id("package-id").build())
+				.lifecycle(createLifecycle())
+				.droplet(Droplet.builder().id("droplet-id").build())
+				.createdAt("DATETIME")
+				.id("build-id")
+				.build()));
+
+		when(deploymentsV3.create(any()))
+			.thenReturn(Mono.just(CreateDeploymentResponse
+				.builder()
+				.state(DeploymentState.DEPLOYED)
+				.createdAt("DATETIME")
+				.id("deployment-id")
+				.build()));
+		when(deploymentsV3.get(any()))
+			.thenReturn(Mono.just(GetDeploymentResponse
+				.builder()
+				.state(DeploymentState.DEPLOYED)
+				.createdAt("DATETIME")
+				.id("deployment-id")
+				.build()));
+
+		UpdateApplicationRequest request =
+			UpdateApplicationRequest
+				.builder()
+				.name(APP_NAME)
+				.path(APP_PATH)
+				.build();
+
+		StepVerifier.create(appDeployer.update(request))
+					.assertNext(response -> assertThat(response.getName()).isEqualTo(APP_NAME))
+					.verifyComplete();
+	}
+
+	private ApplicationDetail createApplicationDetail() {
+		return ApplicationDetail
+			.builder()
+			.id("app-id")
+			.stack("")
+			.diskQuota(512)
+			.instances(1)
+			.memoryLimit(512)
+			.name("app")
+			.requestedState("STARTED")
+			.runningInstances(1)
+			.build();
+	}
+
+	private static Lifecycle createLifecycle() {
+		return Lifecycle.builder().data(BuildpackData.builder().build()).type(LifecycleType.BUILDPACK).build();
+	}
+
+}

spring-cloud-app-broker-deployer/src/main/java/org/springframework/cloud/appbroker/deployer/AppDeployer.java

@@ -24,6 +24,10 @@ public interface AppDeployer {
 		return Mono.empty();
 	}
 
+	default Mono<UpdateApplicationResponse> update(UpdateApplicationRequest request) {
+		return Mono.empty();
+	}
+
 	default Mono<UndeployApplicationResponse> undeploy(UndeployApplicationRequest request) {
 		return Mono.empty();
 	}

spring-cloud-app-broker-deployer/src/main/java/org/springframework/cloud/appbroker/deployer/UpdateApplicationRequest.java

@@ -0,0 +1,135 @@
+/*
+ * Copyright 2002-2018 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.cloud.appbroker.deployer;
+
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public class UpdateApplicationRequest {
+
+	private final String name;
+
+	private final String path;
+
+	private final Map<String, String> properties;
+
+	private final Map<String, Object> environment;
+
+	private final List<String> services;
+
+	UpdateApplicationRequest(String name, String path, Map<String, String> properties,
+	                         Map<String, Object> environment, List<String> services) {
+		this.name = name;
+		this.path = path;
+		this.properties = properties;
+		this.environment = environment;
+		this.services = services;
+	}
+
+	public static DeployApplicationRequestBuilder builder() {
+		return new DeployApplicationRequestBuilder();
+	}
+
+	public String getName() {
+		return name;
+	}
+
+	public String getPath() {
+		return path;
+	}
+
+	public Map<String, String> getProperties() {
+		return properties;
+	}
+
+	public Map<String, Object> getEnvironment() {
+		return environment;
+	}
+
+	public List<String> getServices() {
+		return services;
+	}
+
+	public static class DeployApplicationRequestBuilder {
+
+		private String name;
+
+		private String path;
+
+		private final Map<String, String> properties = new HashMap<>();
+
+		private final Map<String, Object> environment = new HashMap<>();
+
+		private final List<String> services = new ArrayList<>();
+
+		DeployApplicationRequestBuilder() {
+		}
+
+		public DeployApplicationRequestBuilder name(String name) {
+			this.name = name;
+			return this;
+		}
+
+		public DeployApplicationRequestBuilder path(String path) {
+			this.path = path;
+			return this;
+		}
+
+		public DeployApplicationRequestBuilder property(String key, String value) {
+			this.properties.put(key, value);
+			return this;
+		}
+
+		public DeployApplicationRequestBuilder properties(Map<String, String> properties) {
+			if (properties == null) {
+				return this;
+			}
+			this.properties.putAll(properties);
+			return this;
+		}
+
+		public DeployApplicationRequestBuilder environment(String key, String value) {
+			this.environment.put(key, value);
+			return this;
+		}
+
+		public DeployApplicationRequestBuilder environment(Map<String, Object> environment) {
+			if (environment == null) {
+				return this;
+			}
+			this.environment.putAll(environment);
+			return this;
+		}
+
+		public DeployApplicationRequestBuilder service(String service) {
+			this.services.add(service);
+			return this;
+		}
+
+		public DeployApplicationRequestBuilder services(List<String> services) {
+			this.services.addAll(services);
+			return this;
+		}
+
+		public UpdateApplicationRequest build() {
+			return new UpdateApplicationRequest(name, path, properties, environment, services);
+		}
+	}
+
+}

spring-cloud-app-broker-deployer/src/main/java/org/springframework/cloud/appbroker/deployer/UpdateApplicationResponse.java

@@ -0,0 +1,52 @@
+/*
+ * Copyright 2002-2018 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.springframework.cloud.appbroker.deployer;
+
+public class UpdateApplicationResponse {
+
+	private final String name;
+
+	UpdateApplicationResponse(String name) {
+		this.name = name;
+	}
+
+	public static DeployApplicationResponseBuilder builder() {
+		return new DeployApplicationResponseBuilder();
+	}
+
+	public String getName() {
+		return name;
+	}
+
+	public static class DeployApplicationResponseBuilder {
+
+		private String name;
+
+		DeployApplicationResponseBuilder() {
+		}
+
+		public DeployApplicationResponseBuilder name(String name) {
+			this.name = name;
+			return this;
+		}
+
+		public UpdateApplicationResponse build() {
+			return new UpdateApplicationResponse(name);
+		}
+
+	}
+}

spring-cloud-app-broker-integration-tests/src/test/java/org.springframework.cloud.appbroker/integration/fixtures/CloudControllerStubFixture.java

@@ -189,6 +189,15 @@ public class CloudControllerStubFixture extends WiremockStubFixture {
 		stubAppAfterCreation(appName, host);
 	}
 
+	public void stubUpdateApp(final String appName) {
+		stubGetApp(appName);
+		stubUpdateEnvironment(appName);
+		stubCreatePackage(appName);
+		stubCreateBuild(appName);
+		stubCreateDeployment(appName);
+		stubAppAfterCreation(appName, appName);
+	}
+
 	private void stubAppAfterCreation(String appName, String host) {
 		stubMapRouteToApp(appName, host);
 		stubUploadAppBits(appName);
@@ -196,9 +205,69 @@ public class CloudControllerStubFixture extends WiremockStubFixture {
 		stubCheckAppState(appName);
 	}
 
-	public void stubUpdateApp(final String appName, ContentPattern<?>... appMetadataPatterns) {
-		stubUpdateAppMetadata(appName, appMetadataPatterns);
-		stubAppAfterCreation(appName, appName);
+	private void stubGetApp(String appName) {
+		stubFor(get(urlPathEqualTo("/v2/apps/" + appGuid(appName)))
+			.willReturn(ok()
+				.withBody(cc("get-app-STARTED",
+					replace("@name", appName),
+					replace("@guid", appGuid(appName))))));
+		stubFor(get(urlPathEqualTo("/v2/apps/" + appGuid(appName) + "/routes?page=1"))
+			.willReturn(ok()
+				.withBody(cc("list-routes",
+					replace("@name", appName),
+					replace("@guid", appGuid(appName))))));
+	}
+
+	private void stubUpdateEnvironment(String appName) {
+		stubFor(put(urlPathEqualTo("/v2/apps/" + appGuid(appName)))
+			.withRequestBody(matchingJsonPath("$.[?(@.environment_json)]"))
+			.willReturn(ok()
+				.withBody(cc("get-app-STARTED",
+					replace("@name", appName),
+					replace("@guid", appGuid(appName))))));
+	}
+
+	private void stubCreatePackage(String appName) {
+		stubFor(post(urlPathEqualTo("/v3/packages"))
+			.withRequestBody(
+				matchingJsonPath("$.[?(@.relationships.app.data.guid == '" + appGuid(appName) + "')]"))
+			.willReturn(ok()
+				.withBody(cc("get-package-READY",
+					replace("@guid", appGuid(appName))))));
+		stubFor(post(urlPathEqualTo("/v3/packages/" + appGuid(appName) + "/upload"))
+			.willReturn(ok()
+				.withBody(cc("get-package-READY",
+					replace("@guid", appGuid(appName))))));
+		stubFor(get(urlPathEqualTo("/v3/packages/" + appGuid(appName)))
+			.willReturn(ok()
+				.withBody(cc("get-package-READY",
+					replace("@guid", appGuid(appName))))));
+	}
+
+	private void stubCreateBuild(String appName) {
+		stubFor(get(urlPathEqualTo("/v3/builds"))
+			.willReturn(ok()
+				.withBody(cc("get-build-STAGED",
+					replace("@guid", appGuid(appName))))));
+		stubFor(post(urlPathEqualTo("/v3/builds"))
+			.willReturn(ok()
+				.withBody(cc("get-build-STAGED",
+					replace("@guid", appGuid(appName))))));
+		stubFor(get(urlPathEqualTo("/v3/builds/" + appGuid(appName)))
+			.willReturn(ok()
+				.withBody(cc("get-build-STAGED",
+					replace("@guid", appGuid(appName))))));
+	}
+
+	private void stubCreateDeployment(String appName) {
+		stubFor(post(urlPathEqualTo("/v3/deployments"))
+			.willReturn(ok()
+				.withBody(cc("get-deployment-DEPLOYED",
+					replace("@guid", appGuid(appName))))));
+		stubFor(get(urlPathEqualTo("/v3/deployments/" + appGuid(appName)))
+			.willReturn(ok()
+				.withBody(cc("get-deployment-DEPLOYED",
+					replace("@guid", appGuid(appName))))));
 	}
 
 	private void stubCreateAppMetadata(String appName, ContentPattern<?>... appMetadataPatterns) {
@@ -214,19 +283,6 @@ public class CloudControllerStubFixture extends WiremockStubFixture {
 					replace("@guid", appGuid(appName))))));
 	}
 
-	private void stubUpdateAppMetadata(String appName, ContentPattern<?>... appMetadataPatterns) {
-		MappingBuilder mappingBuilder = put(urlPathEqualTo("/v2/apps/" + appGuid(appName)))
-			.withRequestBody(matchingJsonPath("$.[?(@.name == '" + appName + "')]"));
-		for (ContentPattern<?> appMetadataPattern : appMetadataPatterns) {
-			mappingBuilder.withRequestBody(appMetadataPattern);
-		}
-		stubFor(mappingBuilder
-			.willReturn(ok()
-				.withBody(cc("get-app-STARTED",
-					replace("@name", appName),
-					replace("@guid", appGuid(appName))))));
-	}
-
 	private void stubMapRouteToApp(String appName, String host) {
 		stubFor(get(urlPathEqualTo("/v2/apps/" + appGuid(appName) + "/routes"))
 			.willReturn(ok()

spring-cloud-app-broker-integration-tests/src/test/resources/responses/cloudcontroller/get-build-STAGED.json

@@ -0,0 +1,35 @@
+{
+	"guid": "@guid",
+	"created_at": "2016-03-28T23:39:34Z",
+	"updated_at": "2016-03-28T23:39:47Z",
+	"created_by": {
+		"guid": "3cb4e243-bed4-49d5-8739-f8b45abdec1c",
+		"name": "bill",
+		"email": "bill@example.com"
+	},
+	"state": "STAGED",
+	"error": null,
+	"lifecycle": {
+		"type": "buildpack",
+		"data": {
+			"buildpacks": [
+				"ruby_buildpack"
+			],
+			"stack": "cflinuxfs2"
+		}
+	},
+	"package": {
+		"guid": "8e4da443-f255-499c-8b47-b3729b5b7432"
+	},
+	"droplet": {
+		"guid": "1e1186e7-d803-4c46-b9d6-5c81e50fe55a"
+	},
+	"links": {
+		"self": {
+			"href": "https://api.example.org/v3/builds/585bc3c1-3743-497d-88b0-403ad6b56d16"
+		},
+		"app": {
+			"href": "https://api.example.org/v3/apps/7b34f1cf-7e73-428a-bb5a-8a17a8058396"
+		}
+	}
+}

spring-cloud-app-broker-integration-tests/src/test/resources/responses/cloudcontroller/get-deployment-DEPLOYED.json

@@ -0,0 +1,37 @@
+{
+	"guid": "@guid",
+	"state": "DEPLOYED",
+	"droplet": {
+		"guid": "44ccfa61-dbcf-4a0d-82fe-f668e9d2a962"
+	},
+	"previous_droplet": {
+		"guid": "cc6bc315-bd06-49ce-92c2-bc3ad45268c2"
+	},
+	"new_processes": [
+		{
+			"guid": "fd5d3e60-f88c-4c37-b1ae-667cfc65a856",
+			"type": "web-deployment-59c3d133-2b83-46f3-960e-7765a129aea4"
+		}
+	],
+	"revision": {
+		"guid": "56126cba-656a-4eba-a81e-7e9951b2df57",
+		"version": 1
+	},
+	"created_at": "2018-04-25T22:42:10Z",
+	"updated_at": "2018-04-25T22:42:10Z",
+	"relationships": {
+		"app": {
+			"data": {
+				"guid": "305cea31-5a44-45ca-b51b-e89c7a8ef8b2"
+			}
+		}
+	},
+	"links": {
+		"self": {
+			"href": "https://api.example.org/v3/deployments/59c3d133-2b83-46f3-960e-7765a129aea4"
+		},
+		"app": {
+			"href": "https://api.example.org/v3/apps/305cea31-5a44-45ca-b51b-e89c7a8ef8b2"
+		}
+	}
+}

spring-cloud-app-broker-integration-tests/src/test/resources/responses/cloudcontroller/get-package-READY.json

@@ -0,0 +1,30 @@
+{
+	"guid": "@guid",
+	"type": "bits",
+	"data": {
+		"checksum": {
+			"type": "sha256",
+			"value": null
+		},
+		"error": null
+	},
+	"state": "READY",
+	"created_at": "2015-11-13T17:02:56Z",
+	"updated_at": "2016-06-08T16:41:26Z",
+	"links": {
+		"self": {
+			"href": "https://api.example.org/v3/packages/44f7c078-0934-470f-9883-4fcddc5b8f13"
+		},
+		"upload": {
+			"href": "https://api.example.org/v3/packages/44f7c078-0934-470f-9883-4fcddc5b8f13/upload",
+			"method": "POST"
+		},
+		"download": {
+			"href": "https://api.example.org/v3/packages/44f7c078-0934-470f-9883-4fcddc5b8f13/download",
+			"method": "GET"
+		},
+		"app": {
+			"href": "https://api.example.org/v3/apps/1d3bf0ec-5806-43c4-b64e-8364dba1086a"
+		}
+	}
+}