From 0f4e91c8bf9ef685a6483dbf39bfd27f899059f4 Mon Sep 17 00:00:00 2001 From: Mark Paluch Date: Fri, 2 Aug 2019 14:54:04 +0200 Subject: [PATCH] #124 - Introduce parallel build executor. --- release-tools/readme.md | 2 - .../data/release/build/BuildExecutor.java | 168 ++++++++++++++++++ .../data/release/build/BuildOperations.java | 69 ++----- .../data/release/build/MavenBuildSystem.java | 4 +- .../data/release/build/MavenProperties.java | 1 + .../data/release/build/MavenRuntime.java | 11 +- .../release/deployment/ArtifactoryClient.java | 6 +- .../deployment/DeploymentOperations.java | 19 ++ .../data/release/model/Module.java | 7 +- .../data/release/model/ModuleIteration.java | 3 +- .../data/release/model/ProjectAware.java | 24 +++ 11 files changed, 244 insertions(+), 70 deletions(-) create mode 100644 release-tools/src/main/java/org/springframework/data/release/build/BuildExecutor.java create mode 100644 release-tools/src/main/java/org/springframework/data/release/model/ProjectAware.java diff --git a/release-tools/readme.md b/release-tools/readme.md index 118cce3..969945d 100644 --- a/release-tools/readme.md +++ b/release-tools/readme.md @@ -2,8 +2,6 @@ ### Infrastructure requirements -- Pivotal VPN account -- User account (for SCP access) on `docs.af.pivotal.io`. Needs to be registered within local `settings.xml` for a server named `static-dot-s2`. - Credentials for `buildmaster` accounts on https://repo.spring.io. - Credentials for https://oss.sonatype.org (to deploy and promote GA and service releases, need deployment permissions for `org.springframework.data`) in `settings.xml` for server with id `sonatype`. diff --git a/release-tools/src/main/java/org/springframework/data/release/build/BuildExecutor.java b/release-tools/src/main/java/org/springframework/data/release/build/BuildExecutor.java new file mode 100644 index 0000000..a865937 --- /dev/null +++ b/release-tools/src/main/java/org/springframework/data/release/build/BuildExecutor.java @@ -0,0 +1,168 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.data.release.build; + +import lombok.NonNull; + +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ArrayBlockingQueue; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; +import java.util.function.BiFunction; +import java.util.function.Supplier; +import java.util.stream.Collectors; + +import javax.annotation.PreDestroy; + +import org.springframework.data.release.Streamable; +import org.springframework.data.release.model.Project; +import org.springframework.data.release.model.ProjectAware; +import org.springframework.plugin.core.PluginRegistry; +import org.springframework.stereotype.Component; +import org.springframework.util.Assert; + +import com.google.common.util.concurrent.MoreExecutors; + +/** + * Build executor service. + * + * @author Mark Paluch + */ +@Component +class BuildExecutor { + + private final @NonNull PluginRegistry buildSystems; + private final MavenProperties mavenProperties; + private final ExecutorService executor; + + public BuildExecutor(PluginRegistry buildSystems, MavenProperties mavenProperties) { + + this.buildSystems = buildSystems; + this.mavenProperties = mavenProperties; + + if (this.mavenProperties.isParllelize()) { + int processors = Runtime.getRuntime().availableProcessors(); + int parallelity = Math.max(2, processors - 2); + executor = new ThreadPoolExecutor(parallelity, parallelity, 10, TimeUnit.MINUTES, new ArrayBlockingQueue<>(256)); + } else { + executor = MoreExecutors.newDirectExecutorService(); + } + } + + @PreDestroy + public void shutdown() { + executor.shutdown(); + } + + /** + * Selects the build system for each module contained in the given iteration and executes the given function for it + * considering pre-requites, honoring the order. + * + * @param iteration must not be {@literal null}. + * @param function must not be {@literal null}. + * @return + */ + public List doWithBuildSystemOrdered(Streamable iteration, + BiFunction function) { + return doWithBuildSystem(iteration, function, true); + } + + /** + * Selects the build system for each module contained in the given iteration and executes the given function for it + * considering pre-requites, without considering the execution order. + * + * @param iteration must not be {@literal null}. + * @param function must not be {@literal null}. + * @return + */ + public List doWithBuildSystemAnyOrder(Streamable iteration, + BiFunction function) { + return doWithBuildSystem(iteration, function, false); + } + + private List doWithBuildSystem(Streamable iteration, + BiFunction function, boolean considerDependencyOrder) { + + Map> results = new ConcurrentHashMap<>(); + + // Add here projects that should be skipped because of a partial deployment to e.g. Sonatype. + Set skip = new HashSet<>(Arrays.asList()); + + skip.forEach(it -> results.put(it, CompletableFuture.completedFuture(null))); + + for (M moduleIteration : iteration) { + + if (skip.contains(moduleIteration.getProject())) { + continue; + } + + if (considerDependencyOrder) { + Set dependencies = moduleIteration.getProject().getDependencies(); + for (Project dependency : dependencies) { + + CompletableFuture futureResult = results.get(dependency); + + if (futureResult == null) { + throw new IllegalStateException("No future result for " + dependency.getName() + ", required by " + + moduleIteration.getProject().getName()); + } + + futureResult.join(); + } + } + + CompletableFuture result = run(moduleIteration, function); + results.put(moduleIteration.getProject(), result); + } + + return iteration.stream()// + .map(module -> results.get(module.getProject()).join()) // + .collect(Collectors.toList()); + } + + private CompletableFuture run(M module, BiFunction function) { + + Assert.notNull(module, "Module must not be null!"); + + CompletableFuture result = new CompletableFuture<>(); + Supplier exception = () -> new IllegalStateException( + String.format("No build system plugin found for project %s!", module.getProject())); + + BuildSystem buildSystem = buildSystems.getPluginFor(module.getProject(), exception); + + Runnable runnable = () -> { + + try { + + result.complete(function.apply(buildSystem, module)); + } catch (Exception e) { + result.completeExceptionally(e); + } + }; + + executor.execute(runnable); + + return result; + } + +} diff --git a/release-tools/src/main/java/org/springframework/data/release/build/BuildOperations.java b/release-tools/src/main/java/org/springframework/data/release/build/BuildOperations.java index 2cbecd8..a1eab1f 100644 --- a/release-tools/src/main/java/org/springframework/data/release/build/BuildOperations.java +++ b/release-tools/src/main/java/org/springframework/data/release/build/BuildOperations.java @@ -22,10 +22,8 @@ import java.nio.file.Path; import java.util.List; import java.util.function.BiFunction; import java.util.function.Supplier; -import java.util.stream.Collectors; import org.springframework.data.release.deployment.DeploymentInformation; -import org.springframework.data.release.model.Module; import org.springframework.data.release.model.ModuleIteration; import org.springframework.data.release.model.Phase; import org.springframework.data.release.model.Project; @@ -35,6 +33,8 @@ import org.springframework.plugin.core.PluginRegistry; import org.springframework.stereotype.Component; import org.springframework.util.Assert; +import com.google.common.annotations.VisibleForTesting; + /** * @author Oliver Gierke * @author Mark Paluch @@ -45,6 +45,7 @@ public class BuildOperations { private final @NonNull PluginRegistry buildSystems; private final @NonNull MavenProperties properties; + private final @NonNull BuildExecutor executor; /** * Updates all inter-project dependencies based on the given {@link TrainIteration} and release {@link Phase}. @@ -60,7 +61,8 @@ public class BuildOperations { UpdateInformation updateInformation = UpdateInformation.of(iteration, phase); - doWithBuildSystem(iteration, (system, it) -> system.updateProjectDescriptors(it, updateInformation)); + executor.doWithBuildSystemOrdered(iteration, + (system, it) -> system.updateProjectDescriptors(it, updateInformation)); } /** @@ -84,7 +86,7 @@ public class BuildOperations { Assert.notNull(train, "Train must not be null!"); - doWithBuildSystem(train, BuildSystem::triggerDistributionBuild); + executor.doWithBuildSystemAnyOrder(train, BuildSystem::triggerDistributionBuild); } /** @@ -94,7 +96,8 @@ public class BuildOperations { * @return */ public List performRelease(TrainIteration iteration) { - return iteration.stream().map(this::performRelease).collect(Collectors.toList()); + return executor.doWithBuildSystemOrdered(iteration, + (buildSystem, moduleIteration) -> performRelease(moduleIteration)); } /** @@ -118,7 +121,7 @@ public class BuildOperations { Assert.notNull(iteration, "Train iteration must not be null!"); Assert.notNull(phase, "Phase must not be null!"); - doWithBuildSystem(iteration, (system, module) -> system.prepareVersion(module, phase)); + executor.doWithBuildSystemOrdered(iteration, (system, module) -> system.prepareVersion(module, phase)); } /** @@ -128,6 +131,7 @@ public class BuildOperations { * @param phase must not be {@literal null}. * @return */ + @VisibleForTesting public ModuleIteration prepareVersion(ModuleIteration iteration, Phase phase) { Assert.notNull(iteration, "Module iteration must not be null!"); @@ -156,7 +160,7 @@ public class BuildOperations { } /** - * Triggers a nomarl build for the given {@link ModuleIteration}. + * Triggers a normal build for the given {@link ModuleIteration}. * * @param module must not be {@literal null}. * @return @@ -174,38 +178,7 @@ public class BuildOperations { Assert.notNull(iteration, "Train iteration must not be null!"); - doWithBuildSystem(iteration, (system, module) -> system.triggerPreReleaseCheck(module)); - } - - /** - * Selects the build system for each {@link ModuleIteration} contained in the given {@link TrainIteration} and - * executes the given function for it. - * - * @param iteration must not be {@literal null}. - * @param function must not be {@literal null}. - * @return - */ - private List doWithBuildSystem(TrainIteration iteration, - BiFunction function) { - - return iteration.stream()// - .map(module -> doWithBuildSystem(module, function))// - .collect(Collectors.toList()); - } - - /** - * Selects the build system for each {@link Module} contained in the given {@link Train} and executes the given - * function for it. - * - * @param train must not be {@literal null}. - * @param function must not be {@literal null}. - * @return - */ - private List doWithBuildSystem(Train train, BiFunction function) { - - return train.stream()// - .map(module -> doWithBuildSystem(module, function))// - .collect(Collectors.toList()); + executor.doWithBuildSystemAnyOrder(iteration, BuildSystem::triggerPreReleaseCheck); } /** @@ -225,22 +198,4 @@ public class BuildOperations { return function.apply(buildSystems.getPluginFor(module.getProject(), exception), module); } - - /** - * Selects the build system for the module contained in the given {@link Module} and executes the given function with - * it. - * - * @param module must not be {@literal null}. - * @param function must not be {@literal null}. - * @return - */ - private T doWithBuildSystem(Module module, BiFunction function) { - - Assert.notNull(module, "ModuleIteration must not be null!"); - - Supplier exception = () -> new IllegalStateException( - String.format("No build system plugin found for project %s!", module.getProject())); - - return function.apply(buildSystems.getPluginFor(module.getProject(), exception), module); - } } diff --git a/release-tools/src/main/java/org/springframework/data/release/build/MavenBuildSystem.java b/release-tools/src/main/java/org/springframework/data/release/build/MavenBuildSystem.java index 9a5a965..380c905 100644 --- a/release-tools/src/main/java/org/springframework/data/release/build/MavenBuildSystem.java +++ b/release-tools/src/main/java/org/springframework/data/release/build/MavenBuildSystem.java @@ -117,14 +117,14 @@ class MavenBuildSystem implements BuildSystem { mvn.execute(project, CommandLine.of(Goal.CLEAN, Goal.DEPLOY, // SKIP_TESTS, profile("distribute"), Argument.of("-B"), arg("artifactory.server").withValue(properties.getServer().getUri()), - arg("distribution-repository").withValue(properties.getDistributionRepository()), + arg("artifactory.distribution-repository").withValue(properties.getDistributionRepository()), arg("artifactory.username").withValue(properties.getUsername()), arg("artifactory.password").withValue(properties.getPassword()))); mvn.execute(project, CommandLine.of(Goal.CLEAN, Goal.DEPLOY, // SKIP_TESTS, profile("distribute-schema"), Argument.of("-B"), arg("artifactory.server").withValue(properties.getServer().getUri()), - arg("distribution-repository").withValue(properties.getDistributionRepository()), + arg("artifactory.distribution-repository").withValue(properties.getDistributionRepository()), arg("artifactory.username").withValue(properties.getUsername()), arg("artifactory.password").withValue(properties.getPassword()))); diff --git a/release-tools/src/main/java/org/springframework/data/release/build/MavenProperties.java b/release-tools/src/main/java/org/springframework/data/release/build/MavenProperties.java index 95edbc4..15cce6d 100644 --- a/release-tools/src/main/java/org/springframework/data/release/build/MavenProperties.java +++ b/release-tools/src/main/java/org/springframework/data/release/build/MavenProperties.java @@ -41,6 +41,7 @@ class MavenProperties { private File localRepository; private Map plugins; private boolean consoleLogger = true; + private boolean parllelize = false; /** * Configures the local Maven repository location to use. In case the given folder does not already exists it's diff --git a/release-tools/src/main/java/org/springframework/data/release/build/MavenRuntime.java b/release-tools/src/main/java/org/springframework/data/release/build/MavenRuntime.java index 2a63249..4407308 100644 --- a/release-tools/src/main/java/org/springframework/data/release/build/MavenRuntime.java +++ b/release-tools/src/main/java/org/springframework/data/release/build/MavenRuntime.java @@ -94,9 +94,15 @@ class MavenRuntime { InvocationResult result = invoker.execute(request); if (result.getExitCode() != 0) { - throw new RuntimeException(result.getExecutionException()); + logger.warn(project, "Failed execution mvn %s", arguments.toString()); + + throw new IllegalStateException("Failed execution mvn " + arguments.toString(), result.getExecutionException()); } + logger.log(project, "Successful execution mvn %s", arguments.toString()); } catch (Exception e) { + if (e instanceof RuntimeException) { + throw (RuntimeException) e; + } throw new RuntimeException(e); } } @@ -160,7 +166,7 @@ class MavenRuntime { String goalNames = goals.stream().map(CommandLine.Goal::getGoal).collect(Collectors.joining("-")); - String filename = String.format("mvn-%s-%s.log", project.getName(), goalNames); + String filename = String.format("mvn-%s-%s.log", project.getName(), goalNames).replace(':', '.'); try { File file = new File(logsDirectory, filename); @@ -176,7 +182,6 @@ class MavenRuntime { @Override public void info(String message) { printWriter.println(message); - } @Override diff --git a/release-tools/src/main/java/org/springframework/data/release/deployment/ArtifactoryClient.java b/release-tools/src/main/java/org/springframework/data/release/deployment/ArtifactoryClient.java index c516c6a..f80d02f 100644 --- a/release-tools/src/main/java/org/springframework/data/release/deployment/ArtifactoryClient.java +++ b/release-tools/src/main/java/org/springframework/data/release/deployment/ArtifactoryClient.java @@ -31,8 +31,9 @@ import com.fasterxml.jackson.databind.ObjectMapper; /** * A client to interact with Artifactory. - * + * * @author Oliver Gierke + * @author Mark Paluch */ @RequiredArgsConstructor class ArtifactoryClient { @@ -43,7 +44,7 @@ class ArtifactoryClient { /** * Triggers the promotion of the artifacts identified by the given {@link DeploymentInformation}. - * + * * @param information must not be {@literal null}. */ public void promote(DeploymentInformation information) { @@ -78,6 +79,7 @@ class ArtifactoryClient { } catch (HttpClientErrorException o_O) { handle("Authentication verification failed!", o_O); + throw new IllegalStateException("Authentication verification failed!"); } } diff --git a/release-tools/src/main/java/org/springframework/data/release/deployment/DeploymentOperations.java b/release-tools/src/main/java/org/springframework/data/release/deployment/DeploymentOperations.java index 2b5ac89..cbf91de 100644 --- a/release-tools/src/main/java/org/springframework/data/release/deployment/DeploymentOperations.java +++ b/release-tools/src/main/java/org/springframework/data/release/deployment/DeploymentOperations.java @@ -25,6 +25,7 @@ import org.springframework.util.Assert; * Deployment functionality. * * @author Oliver Gierke + * @author Mark Paluch */ @Component @RequiredArgsConstructor @@ -54,4 +55,22 @@ public class DeploymentOperations { client.promote(information); } + + /** + * Rolls back the given {@link DeploymentInformation}. + * + * @param information must not be {@literal null}. + */ + public void rollback(DeploymentInformation information) { + + Assert.notNull(information, "DeploymentInformation must not be null!"); + + if (information.getModule().getIteration().isPublic()) { + logger.log(information.getModule(), + "Skipping build rollback as it's a public version and was deployed to Maven Central directly,"); + return; + } + + client.deleteArtifacts(information); + } } diff --git a/release-tools/src/main/java/org/springframework/data/release/model/Module.java b/release-tools/src/main/java/org/springframework/data/release/model/Module.java index 16e1e9a..08977da 100644 --- a/release-tools/src/main/java/org/springframework/data/release/model/Module.java +++ b/release-tools/src/main/java/org/springframework/data/release/model/Module.java @@ -21,9 +21,10 @@ import org.springframework.util.Assert; /** * @author Oliver Gierke + * @author Mark Paluch */ @Value -public class Module implements VersionAware, Comparable { +public class Module implements VersionAware, ProjectAware, Comparable { private final Project project; private final Version version; @@ -59,7 +60,7 @@ public class Module implements VersionAware, Comparable { /** * Returns whether the current {@link Module} refers to the same project as the given one. - * + * * @param module must not be {@literal null}. * @return */ @@ -67,7 +68,7 @@ public class Module implements VersionAware, Comparable { return this.project.equals(module.project); } - /* + /* * (non-Javadoc) * @see java.lang.Comparable#compareTo(java.lang.Object) */ diff --git a/release-tools/src/main/java/org/springframework/data/release/model/ModuleIteration.java b/release-tools/src/main/java/org/springframework/data/release/model/ModuleIteration.java index 5e556b8..4d0397a 100644 --- a/release-tools/src/main/java/org/springframework/data/release/model/ModuleIteration.java +++ b/release-tools/src/main/java/org/springframework/data/release/model/ModuleIteration.java @@ -22,10 +22,11 @@ import lombok.RequiredArgsConstructor; /** * @author Oliver Gierke + * @author Mark Paluch */ @RequiredArgsConstructor(access = AccessLevel.PACKAGE) @EqualsAndHashCode -public class ModuleIteration implements IterationVersion { +public class ModuleIteration implements IterationVersion, ProjectAware{ private final @Getter Module module; private final @Getter TrainIteration trainIteration; diff --git a/release-tools/src/main/java/org/springframework/data/release/model/ProjectAware.java b/release-tools/src/main/java/org/springframework/data/release/model/ProjectAware.java new file mode 100644 index 0000000..0d8bf3f --- /dev/null +++ b/release-tools/src/main/java/org/springframework/data/release/model/ProjectAware.java @@ -0,0 +1,24 @@ +/* + * Copyright 2019 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.data.release.model; + +/** + * @author Mark Paluch + */ +public interface ProjectAware { + + Project getProject(); +}