#124 - Introduce parallel build executor.

release-tools/readme.md

@@ -2,8 +2,6 @@
 
 ### Infrastructure requirements
 
-- Pivotal VPN account
-- User account (for SCP access) on `docs.af.pivotal.io`. Needs to be registered within local `settings.xml` for a server named `static-dot-s2`.
 - Credentials for `buildmaster` accounts on https://repo.spring.io.
 - Credentials for https://oss.sonatype.org (to deploy and promote GA and service releases, need deployment permissions for `org.springframework.data`) in `settings.xml` for server with id `sonatype`.
 

release-tools/src/main/java/org/springframework/data/release/build/BuildExecutor.java

@@ -0,0 +1,168 @@
+/*
+ * Copyright 2019 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.data.release.build;
+
+import lombok.NonNull;
+
+import java.util.Arrays;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.function.BiFunction;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
+
+import javax.annotation.PreDestroy;
+
+import org.springframework.data.release.Streamable;
+import org.springframework.data.release.model.Project;
+import org.springframework.data.release.model.ProjectAware;
+import org.springframework.plugin.core.PluginRegistry;
+import org.springframework.stereotype.Component;
+import org.springframework.util.Assert;
+
+import com.google.common.util.concurrent.MoreExecutors;
+
+/**
+ * Build executor service.
+ *
+ * @author Mark Paluch
+ */
+@Component
+class BuildExecutor {
+
+	private final @NonNull PluginRegistry<BuildSystem, Project> buildSystems;
+	private final MavenProperties mavenProperties;
+	private final ExecutorService executor;
+
+	public BuildExecutor(PluginRegistry<BuildSystem, Project> buildSystems, MavenProperties mavenProperties) {
+
+		this.buildSystems = buildSystems;
+		this.mavenProperties = mavenProperties;
+
+		if (this.mavenProperties.isParllelize()) {
+			int processors = Runtime.getRuntime().availableProcessors();
+			int parallelity = Math.max(2, processors - 2);
+			executor = new ThreadPoolExecutor(parallelity, parallelity, 10, TimeUnit.MINUTES, new ArrayBlockingQueue<>(256));
+		} else {
+			executor = MoreExecutors.newDirectExecutorService();
+		}
+	}
+
+	@PreDestroy
+	public void shutdown() {
+		executor.shutdown();
+	}
+
+	/**
+	 * Selects the build system for each module contained in the given iteration and executes the given function for it
+	 * considering pre-requites, honoring the order.
+	 *
+	 * @param iteration must not be {@literal null}.
+	 * @param function must not be {@literal null}.
+	 * @return
+	 */
+	public <T, M extends ProjectAware> List<T> doWithBuildSystemOrdered(Streamable<M> iteration,
+			BiFunction<BuildSystem, M, T> function) {
+		return doWithBuildSystem(iteration, function, true);
+	}
+
+	/**
+	 * Selects the build system for each module contained in the given iteration and executes the given function for it
+	 * considering pre-requites, without considering the execution order.
+	 *
+	 * @param iteration must not be {@literal null}.
+	 * @param function must not be {@literal null}.
+	 * @return
+	 */
+	public <T, M extends ProjectAware> List<T> doWithBuildSystemAnyOrder(Streamable<M> iteration,
+			BiFunction<BuildSystem, M, T> function) {
+		return doWithBuildSystem(iteration, function, false);
+	}
+
+	private <T, M extends ProjectAware> List<T> doWithBuildSystem(Streamable<M> iteration,
+			BiFunction<BuildSystem, M, T> function, boolean considerDependencyOrder) {
+
+		Map<Project, CompletableFuture<T>> results = new ConcurrentHashMap<>();
+
+		// Add here projects that should be skipped because of a partial deployment to e.g. Sonatype.
+		Set<Project> skip = new HashSet<>(Arrays.asList());
+
+		skip.forEach(it -> results.put(it, CompletableFuture.completedFuture(null)));
+
+		for (M moduleIteration : iteration) {
+
+			if (skip.contains(moduleIteration.getProject())) {
+				continue;
+			}
+
+			if (considerDependencyOrder) {
+				Set<Project> dependencies = moduleIteration.getProject().getDependencies();
+				for (Project dependency : dependencies) {
+
+					CompletableFuture<T> futureResult = results.get(dependency);
+
+					if (futureResult == null) {
+						throw new IllegalStateException("No future result for " + dependency.getName() + ", required by "
+								+ moduleIteration.getProject().getName());
+					}
+
+					futureResult.join();
+				}
+			}
+
+			CompletableFuture<T> result = run(moduleIteration, function);
+			results.put(moduleIteration.getProject(), result);
+		}
+
+		return iteration.stream()//
+				.map(module -> results.get(module.getProject()).join()) //
+				.collect(Collectors.toList());
+	}
+
+	private <T, M extends ProjectAware> CompletableFuture<T> run(M module, BiFunction<BuildSystem, M, T> function) {
+
+		Assert.notNull(module, "Module must not be null!");
+
+		CompletableFuture<T> result = new CompletableFuture<>();
+		Supplier<IllegalStateException> exception = () -> new IllegalStateException(
+				String.format("No build system plugin found for project %s!", module.getProject()));
+
+		BuildSystem buildSystem = buildSystems.getPluginFor(module.getProject(), exception);
+
+		Runnable runnable = () -> {
+
+			try {
+
+				result.complete(function.apply(buildSystem, module));
+			} catch (Exception e) {
+				result.completeExceptionally(e);
+			}
+		};
+
+		executor.execute(runnable);
+
+		return result;
+	}
+
+}

release-tools/src/main/java/org/springframework/data/release/build/BuildOperations.java

@@ -22,10 +22,8 @@ import java.nio.file.Path;
 import java.util.List;
 import java.util.function.BiFunction;
 import java.util.function.Supplier;
-import java.util.stream.Collectors;
 
 import org.springframework.data.release.deployment.DeploymentInformation;
-import org.springframework.data.release.model.Module;
 import org.springframework.data.release.model.ModuleIteration;
 import org.springframework.data.release.model.Phase;
 import org.springframework.data.release.model.Project;
@@ -35,6 +33,8 @@ import org.springframework.plugin.core.PluginRegistry;
 import org.springframework.stereotype.Component;
 import org.springframework.util.Assert;
 
+import com.google.common.annotations.VisibleForTesting;
+
 /**
  * @author Oliver Gierke
  * @author Mark Paluch
@@ -45,6 +45,7 @@ public class BuildOperations {
 
 	private final @NonNull PluginRegistry<BuildSystem, Project> buildSystems;
 	private final @NonNull MavenProperties properties;
+	private final @NonNull BuildExecutor executor;
 
 	/**
 	 * Updates all inter-project dependencies based on the given {@link TrainIteration} and release {@link Phase}.
@@ -60,7 +61,8 @@ public class BuildOperations {
 
 		UpdateInformation updateInformation = UpdateInformation.of(iteration, phase);
 
-		doWithBuildSystem(iteration, (system, it) -> system.updateProjectDescriptors(it, updateInformation));
+		executor.doWithBuildSystemOrdered(iteration,
+				(system, it) -> system.updateProjectDescriptors(it, updateInformation));
 	}
 
 	/**
@@ -84,7 +86,7 @@ public class BuildOperations {
 
 		Assert.notNull(train, "Train must not be null!");
 
-		doWithBuildSystem(train, BuildSystem::triggerDistributionBuild);
+		executor.doWithBuildSystemAnyOrder(train, BuildSystem::triggerDistributionBuild);
 	}
 
 	/**
@@ -94,7 +96,8 @@ public class BuildOperations {
 	 * @return
 	 */
 	public List<DeploymentInformation> performRelease(TrainIteration iteration) {
-		return iteration.stream().map(this::performRelease).collect(Collectors.toList());
+		return executor.doWithBuildSystemOrdered(iteration,
+				(buildSystem, moduleIteration) -> performRelease(moduleIteration));
 	}
 
 	/**
@@ -118,7 +121,7 @@ public class BuildOperations {
 		Assert.notNull(iteration, "Train iteration must not be null!");
 		Assert.notNull(phase, "Phase must not be null!");
 
-		doWithBuildSystem(iteration, (system, module) -> system.prepareVersion(module, phase));
+		executor.doWithBuildSystemOrdered(iteration, (system, module) -> system.prepareVersion(module, phase));
 	}
 
 	/**
@@ -128,6 +131,7 @@ public class BuildOperations {
 	 * @param phase must not be {@literal null}.
 	 * @return
 	 */
+	@VisibleForTesting
 	public ModuleIteration prepareVersion(ModuleIteration iteration, Phase phase) {
 
 		Assert.notNull(iteration, "Module iteration must not be null!");
@@ -156,7 +160,7 @@ public class BuildOperations {
 	}
 
 	/**
-	 * Triggers a nomarl build for the given {@link ModuleIteration}.
+	 * Triggers a normal build for the given {@link ModuleIteration}.
 	 *
 	 * @param module must not be {@literal null}.
 	 * @return
@@ -174,38 +178,7 @@ public class BuildOperations {
 
 		Assert.notNull(iteration, "Train iteration must not be null!");
 
-		doWithBuildSystem(iteration, (system, module) -> system.triggerPreReleaseCheck(module));
+		executor.doWithBuildSystemAnyOrder(iteration, BuildSystem::triggerPreReleaseCheck);
-	}
-
-	/**
-	 * Selects the build system for each {@link ModuleIteration} contained in the given {@link TrainIteration} and
-	 * executes the given function for it.
-	 *
-	 * @param iteration must not be {@literal null}.
-	 * @param function must not be {@literal null}.
-	 * @return
-	 */
-	private <T> List<T> doWithBuildSystem(TrainIteration iteration,
-			BiFunction<BuildSystem, ModuleIteration, T> function) {
-
-		return iteration.stream()//
-				.map(module -> doWithBuildSystem(module, function))//
-				.collect(Collectors.toList());
-	}
-
-	/**
-	 * Selects the build system for each {@link Module} contained in the given {@link Train} and executes the given
-	 * function for it.
-	 *
-	 * @param train must not be {@literal null}.
-	 * @param function must not be {@literal null}.
-	 * @return
-	 */
-	private <T> List<T> doWithBuildSystem(Train train, BiFunction<BuildSystem, Module, T> function) {
-
-		return train.stream()//
-				.map(module -> doWithBuildSystem(module, function))//
-				.collect(Collectors.toList());
 	}
 
 	/**
@@ -225,22 +198,4 @@ public class BuildOperations {
 
 		return function.apply(buildSystems.getPluginFor(module.getProject(), exception), module);
 	}
-
-	/**
-	 * Selects the build system for the module contained in the given {@link Module} and executes the given function with
-	 * it.
-	 *
-	 * @param module must not be {@literal null}.
-	 * @param function must not be {@literal null}.
-	 * @return
-	 */
-	private <T> T doWithBuildSystem(Module module, BiFunction<BuildSystem, Module, T> function) {
-
-		Assert.notNull(module, "ModuleIteration must not be null!");
-
-		Supplier<IllegalStateException> exception = () -> new IllegalStateException(
-				String.format("No build system plugin found for project %s!", module.getProject()));
-
-		return function.apply(buildSystems.getPluginFor(module.getProject(), exception), module);
-	}
 }

release-tools/src/main/java/org/springframework/data/release/build/MavenBuildSystem.java

@@ -117,14 +117,14 @@ class MavenBuildSystem implements BuildSystem {
 		mvn.execute(project, CommandLine.of(Goal.CLEAN, Goal.DEPLOY, //
 				SKIP_TESTS, profile("distribute"), Argument.of("-B"),
 				arg("artifactory.server").withValue(properties.getServer().getUri()),
-				arg("distribution-repository").withValue(properties.getDistributionRepository()),
+				arg("artifactory.distribution-repository").withValue(properties.getDistributionRepository()),
 				arg("artifactory.username").withValue(properties.getUsername()),
 				arg("artifactory.password").withValue(properties.getPassword())));
 
 		mvn.execute(project, CommandLine.of(Goal.CLEAN, Goal.DEPLOY, //
 				SKIP_TESTS, profile("distribute-schema"), Argument.of("-B"),
 				arg("artifactory.server").withValue(properties.getServer().getUri()),
-				arg("distribution-repository").withValue(properties.getDistributionRepository()),
+				arg("artifactory.distribution-repository").withValue(properties.getDistributionRepository()),
 				arg("artifactory.username").withValue(properties.getUsername()),
 				arg("artifactory.password").withValue(properties.getPassword())));
 

release-tools/src/main/java/org/springframework/data/release/build/MavenProperties.java

@@ -41,6 +41,7 @@ class MavenProperties {
 	private File localRepository;
 	private Map<String, String> plugins;
 	private boolean consoleLogger = true;
+	private boolean parllelize = false;
 
 	/**
 	 * Configures the local Maven repository location to use. In case the given folder does not already exists it's

release-tools/src/main/java/org/springframework/data/release/build/MavenRuntime.java

@@ -94,9 +94,15 @@ class MavenRuntime {
 			InvocationResult result = invoker.execute(request);
 
 			if (result.getExitCode() != 0) {
-				throw new RuntimeException(result.getExecutionException());
+				logger.warn(project, "Failed execution mvn %s", arguments.toString());
+
+				throw new IllegalStateException("Failed execution mvn " + arguments.toString(), result.getExecutionException());
 			}
+			logger.log(project, "Successful execution mvn %s", arguments.toString());
 		} catch (Exception e) {
+			if (e instanceof RuntimeException) {
+				throw (RuntimeException) e;
+			}
 			throw new RuntimeException(e);
 		}
 	}
@@ -160,7 +166,7 @@ class MavenRuntime {
 
 			String goalNames = goals.stream().map(CommandLine.Goal::getGoal).collect(Collectors.joining("-"));
 
-			String filename = String.format("mvn-%s-%s.log", project.getName(), goalNames);
+			String filename = String.format("mvn-%s-%s.log", project.getName(), goalNames).replace(':', '.');
 
 			try {
 				File file = new File(logsDirectory, filename);
@@ -176,7 +182,6 @@ class MavenRuntime {
 		@Override
 		public void info(String message) {
 			printWriter.println(message);
-
 		}
 
 		@Override

release-tools/src/main/java/org/springframework/data/release/deployment/ArtifactoryClient.java

@@ -33,6 +33,7 @@ import com.fasterxml.jackson.databind.ObjectMapper;
  * A client to interact with Artifactory.
  *
  * @author Oliver Gierke
+ * @author Mark Paluch
  */
 @RequiredArgsConstructor
 class ArtifactoryClient {
@@ -78,6 +79,7 @@ class ArtifactoryClient {
 
 		} catch (HttpClientErrorException o_O) {
 			handle("Authentication verification failed!", o_O);
+			throw new IllegalStateException("Authentication verification failed!");
 		}
 	}
 

release-tools/src/main/java/org/springframework/data/release/deployment/DeploymentOperations.java

@@ -25,6 +25,7 @@ import org.springframework.util.Assert;
  * Deployment functionality.
  *
  * @author Oliver Gierke
+ * @author Mark Paluch
  */
 @Component
 @RequiredArgsConstructor
@@ -54,4 +55,22 @@ public class DeploymentOperations {
 
 		client.promote(information);
 	}
+
+	/**
+	 * Rolls back the given {@link DeploymentInformation}.
+	 *
+	 * @param information must not be {@literal null}.
+	 */
+	public void rollback(DeploymentInformation information) {
+
+		Assert.notNull(information, "DeploymentInformation must not be null!");
+
+		if (information.getModule().getIteration().isPublic()) {
+			logger.log(information.getModule(),
+					"Skipping build rollback as it's a public version and was deployed to Maven Central directly,");
+			return;
+		}
+
+		client.deleteArtifacts(information);
+	}
 }

release-tools/src/main/java/org/springframework/data/release/model/Module.java

@@ -21,9 +21,10 @@ import org.springframework.util.Assert;
 
 /**
  * @author Oliver Gierke
+ * @author Mark Paluch
  */
 @Value
-public class Module implements VersionAware, Comparable<Module> {
+public class Module implements VersionAware, ProjectAware, Comparable<Module> {
 
 	private final Project project;
 	private final Version version;

release-tools/src/main/java/org/springframework/data/release/model/ModuleIteration.java

@@ -22,10 +22,11 @@ import lombok.RequiredArgsConstructor;
 
 /**
  * @author Oliver Gierke
+ * @author Mark Paluch
  */
 @RequiredArgsConstructor(access = AccessLevel.PACKAGE)
 @EqualsAndHashCode
-public class ModuleIteration implements IterationVersion {
+public class ModuleIteration implements IterationVersion, ProjectAware{
 
 	private final @Getter Module module;
 	private final @Getter TrainIteration trainIteration;

release-tools/src/main/java/org/springframework/data/release/model/ProjectAware.java

@@ -0,0 +1,24 @@
+/*
+ * Copyright 2019 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.springframework.data.release.model;
+
+/**
+ * @author Mark Paluch
+ */
+public interface ProjectAware {
+
+	Project getProject();
+}