Spring/spring-data-dev-tools
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

#124 - Introduce parallel build executor.

Browse Source
This commit is contained in:
Mark Paluch
2019-08-02 14:54:04 +02:00
parent 63843d6d59
commit 0f4e91c8bf
11 changed files with 244 additions and 70 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
release-tools/readme.md
View File

@@ -2,8 +2,6 @@
### Infrastructure requirements
- Pivotal VPN account
- User account (for SCP access) on `docs.af.pivotal.io`. Needs to be registered within local `settings.xml` for a server named `static-dot-s2`.
- Credentials for `buildmaster` accounts on https://repo.spring.io.
- Credentials for https://oss.sonatype.org (to deploy and promote GA and service releases, need deployment permissions for `org.springframework.data`) in `settings.xml` for server with id `sonatype`.

168
release-tools/src/main/java/org/springframework/data/release/build/BuildExecutor.java Normal file
View File

@@ -0,0 +1,168 @@
/*
* Copyright 2019 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.data.release.build;
import lombok.NonNull;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.function.BiFunction;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import javax.annotation.PreDestroy;
import org.springframework.data.release.Streamable;
import org.springframework.data.release.model.Project;
import org.springframework.data.release.model.ProjectAware;
import org.springframework.plugin.core.PluginRegistry;
import org.springframework.stereotype.Component;
import org.springframework.util.Assert;
import com.google.common.util.concurrent.MoreExecutors;
/**
* Build executor service.
*
* @author Mark Paluch
*/
@Component
class BuildExecutor {
private final @NonNull PluginRegistry<BuildSystem, Project> buildSystems;
private final MavenProperties mavenProperties;
private final ExecutorService executor;
public BuildExecutor(PluginRegistry<BuildSystem, Project> buildSystems, MavenProperties mavenProperties) {
this.buildSystems = buildSystems;
this.mavenProperties = mavenProperties;
if (this.mavenProperties.isParllelize()) {
int processors = Runtime.getRuntime().availableProcessors();
int parallelity = Math.max(2, processors - 2);
executor = new ThreadPoolExecutor(parallelity, parallelity, 10, TimeUnit.MINUTES, new ArrayBlockingQueue<>(256));
} else {
executor = MoreExecutors.newDirectExecutorService();
}
}
@PreDestroy
public void shutdown() {
executor.shutdown();
}
/**
* Selects the build system for each module contained in the given iteration and executes the given function for it
* considering pre-requites, honoring the order.
*
* @param iteration must not be {@literal null}.
* @param function must not be {@literal null}.
* @return
*/
public <T, M extends ProjectAware> List<T> doWithBuildSystemOrdered(Streamable<M> iteration,
BiFunction<BuildSystem, M, T> function) {
return doWithBuildSystem(iteration, function, true);
}
/**
* Selects the build system for each module contained in the given iteration and executes the given function for it
* considering pre-requites, without considering the execution order.
*
* @param iteration must not be {@literal null}.
* @param function must not be {@literal null}.
* @return
*/
public <T, M extends ProjectAware> List<T> doWithBuildSystemAnyOrder(Streamable<M> iteration,
BiFunction<BuildSystem, M, T> function) {
return doWithBuildSystem(iteration, function, false);
}
private <T, M extends ProjectAware> List<T> doWithBuildSystem(Streamable<M> iteration,
BiFunction<BuildSystem, M, T> function, boolean considerDependencyOrder) {
Map<Project, CompletableFuture<T>> results = new ConcurrentHashMap<>();
// Add here projects that should be skipped because of a partial deployment to e.g. Sonatype.
Set<Project> skip = new HashSet<>(Arrays.asList());
skip.forEach(it -> results.put(it, CompletableFuture.completedFuture(null)));
for (M moduleIteration : iteration) {
if (skip.contains(moduleIteration.getProject())) {
continue;
}
if (considerDependencyOrder) {
Set<Project> dependencies = moduleIteration.getProject().getDependencies();
for (Project dependency : dependencies) {
CompletableFuture<T> futureResult = results.get(dependency);
if (futureResult == null) {
throw new IllegalStateException("No future result for " + dependency.getName() + ", required by "
+ moduleIteration.getProject().getName());
}
futureResult.join();
}
}
CompletableFuture<T> result = run(moduleIteration, function);
results.put(moduleIteration.getProject(), result);
}
return iteration.stream()//
.map(module -> results.get(module.getProject()).join()) //
.collect(Collectors.toList());
}
private <T, M extends ProjectAware> CompletableFuture<T> run(M module, BiFunction<BuildSystem, M, T> function) {
Assert.notNull(module, "Module must not be null!");
CompletableFuture<T> result = new CompletableFuture<>();
Supplier<IllegalStateException> exception = () -> new IllegalStateException(
String.format("No build system plugin found for project %s!", module.getProject()));
BuildSystem buildSystem = buildSystems.getPluginFor(module.getProject(), exception);
Runnable runnable = () -> {
try {
result.complete(function.apply(buildSystem, module));
} catch (Exception e) {
result.completeExceptionally(e);
}
};
executor.execute(runnable);
return result;
}
}

69
release-tools/src/main/java/org/springframework/data/release/build/BuildOperations.java
View File

@@ -22,10 +22,8 @@ import java.nio.file.Path;
import java.util.List;
import java.util.function.BiFunction;
import java.util.function.Supplier;
import java.util.stream.Collectors;
import org.springframework.data.release.deployment.DeploymentInformation;
import org.springframework.data.release.model.Module;
import org.springframework.data.release.model.ModuleIteration;
import org.springframework.data.release.model.Phase;
import org.springframework.data.release.model.Project;
@@ -35,6 +33,8 @@ import org.springframework.plugin.core.PluginRegistry;
import org.springframework.stereotype.Component;
import org.springframework.util.Assert;
import com.google.common.annotations.VisibleForTesting;
/**
* @author Oliver Gierke
* @author Mark Paluch
@@ -45,6 +45,7 @@ public class BuildOperations {
private final @NonNull PluginRegistry<BuildSystem, Project> buildSystems;
private final @NonNull MavenProperties properties;
private final @NonNull BuildExecutor executor;
/**
* Updates all inter-project dependencies based on the given {@link TrainIteration} and release {@link Phase}.
@@ -60,7 +61,8 @@ public class BuildOperations {
UpdateInformation updateInformation = UpdateInformation.of(iteration, phase);
doWithBuildSystem(iteration, (system, it) -> system.updateProjectDescriptors(it, updateInformation));
executor.doWithBuildSystemOrdered(iteration,
(system, it) -> system.updateProjectDescriptors(it, updateInformation));
}
/**
@@ -84,7 +86,7 @@ public class BuildOperations {
Assert.notNull(train, "Train must not be null!");
doWithBuildSystem(train, BuildSystem::triggerDistributionBuild);
executor.doWithBuildSystemAnyOrder(train, BuildSystem::triggerDistributionBuild);
}
/**
@@ -94,7 +96,8 @@ public class BuildOperations {
* @return
*/
public List<DeploymentInformation> performRelease(TrainIteration iteration) {
return iteration.stream().map(this::performRelease).collect(Collectors.toList());
return executor.doWithBuildSystemOrdered(iteration,
(buildSystem, moduleIteration) -> performRelease(moduleIteration));
}
/**
@@ -118,7 +121,7 @@ public class BuildOperations {
Assert.notNull(iteration, "Train iteration must not be null!");
Assert.notNull(phase, "Phase must not be null!");
doWithBuildSystem(iteration, (system, module) -> system.prepareVersion(module, phase));
executor.doWithBuildSystemOrdered(iteration, (system, module) -> system.prepareVersion(module, phase));
}
/**
@@ -128,6 +131,7 @@ public class BuildOperations {
* @param phase must not be {@literal null}.
* @return
*/
@VisibleForTesting
public ModuleIteration prepareVersion(ModuleIteration iteration, Phase phase) {
Assert.notNull(iteration, "Module iteration must not be null!");
@@ -156,7 +160,7 @@ public class BuildOperations {
}
/**
* Triggers a nomarl build for the given {@link ModuleIteration}.
* Triggers a normal build for the given {@link ModuleIteration}.
*
* @param module must not be {@literal null}.
* @return
@@ -174,38 +178,7 @@ public class BuildOperations {
Assert.notNull(iteration, "Train iteration must not be null!");
doWithBuildSystem(iteration, (system, module) -> system.triggerPreReleaseCheck(module));
}
/**
* Selects the build system for each {@link ModuleIteration} contained in the given {@link TrainIteration} and
* executes the given function for it.
*
* @param iteration must not be {@literal null}.
* @param function must not be {@literal null}.
* @return
*/
private <T> List<T> doWithBuildSystem(TrainIteration iteration,
BiFunction<BuildSystem, ModuleIteration, T> function) {
return iteration.stream()//
.map(module -> doWithBuildSystem(module, function))//
.collect(Collectors.toList());
}
/**
* Selects the build system for each {@link Module} contained in the given {@link Train} and executes the given
* function for it.
*
* @param train must not be {@literal null}.
* @param function must not be {@literal null}.
* @return
*/
private <T> List<T> doWithBuildSystem(Train train, BiFunction<BuildSystem, Module, T> function) {
return train.stream()//
.map(module -> doWithBuildSystem(module, function))//
.collect(Collectors.toList());
executor.doWithBuildSystemAnyOrder(iteration, BuildSystem::triggerPreReleaseCheck);
}
/**
@@ -225,22 +198,4 @@ public class BuildOperations {
return function.apply(buildSystems.getPluginFor(module.getProject(), exception), module);
}
/**
* Selects the build system for the module contained in the given {@link Module} and executes the given function with
* it.
*
* @param module must not be {@literal null}.
* @param function must not be {@literal null}.
* @return
*/
private <T> T doWithBuildSystem(Module module, BiFunction<BuildSystem, Module, T> function) {
Assert.notNull(module, "ModuleIteration must not be null!");
Supplier<IllegalStateException> exception = () -> new IllegalStateException(
String.format("No build system plugin found for project %s!", module.getProject()));
return function.apply(buildSystems.getPluginFor(module.getProject(), exception), module);
}
}

4
release-tools/src/main/java/org/springframework/data/release/build/MavenBuildSystem.java
View File

@@ -117,14 +117,14 @@ class MavenBuildSystem implements BuildSystem {
mvn.execute(project, CommandLine.of(Goal.CLEAN, Goal.DEPLOY, //
SKIP_TESTS, profile("distribute"), Argument.of("-B"),
arg("artifactory.server").withValue(properties.getServer().getUri()),
arg("distribution-repository").withValue(properties.getDistributionRepository()),
arg("artifactory.distribution-repository").withValue(properties.getDistributionRepository()),
arg("artifactory.username").withValue(properties.getUsername()),
arg("artifactory.password").withValue(properties.getPassword())));
mvn.execute(project, CommandLine.of(Goal.CLEAN, Goal.DEPLOY, //
SKIP_TESTS, profile("distribute-schema"), Argument.of("-B"),
arg("artifactory.server").withValue(properties.getServer().getUri()),
arg("distribution-repository").withValue(properties.getDistributionRepository()),
arg("artifactory.distribution-repository").withValue(properties.getDistributionRepository()),
arg("artifactory.username").withValue(properties.getUsername()),
arg("artifactory.password").withValue(properties.getPassword())));

1
release-tools/src/main/java/org/springframework/data/release/build/MavenProperties.java
View File

@@ -41,6 +41,7 @@ class MavenProperties {
private File localRepository;
private Map<String, String> plugins;
private boolean consoleLogger = true;
private boolean parllelize = false;
/**
* Configures the local Maven repository location to use. In case the given folder does not already exists it's

11
release-tools/src/main/java/org/springframework/data/release/build/MavenRuntime.java
View File

@@ -94,9 +94,15 @@ class MavenRuntime {
InvocationResult result = invoker.execute(request);
if (result.getExitCode() != 0) {
throw new RuntimeException(result.getExecutionException());
logger.warn(project, "Failed execution mvn %s", arguments.toString());
throw new IllegalStateException("Failed execution mvn " + arguments.toString(), result.getExecutionException());
}
logger.log(project, "Successful execution mvn %s", arguments.toString());
} catch (Exception e) {
if (e instanceof RuntimeException) {
throw (RuntimeException) e;
}
throw new RuntimeException(e);
}
}
@@ -160,7 +166,7 @@ class MavenRuntime {
String goalNames = goals.stream().map(CommandLine.Goal::getGoal).collect(Collectors.joining("-"));
String filename = String.format("mvn-%s-%s.log", project.getName(), goalNames);
String filename = String.format("mvn-%s-%s.log", project.getName(), goalNames).replace(':', '.');
try {
File file = new File(logsDirectory, filename);
@@ -176,7 +182,6 @@ class MavenRuntime {
@Override
public void info(String message) {
printWriter.println(message);
}
@Override

6
release-tools/src/main/java/org/springframework/data/release/deployment/ArtifactoryClient.java
View File

@@ -31,8 +31,9 @@ import com.fasterxml.jackson.databind.ObjectMapper;
/**
* A client to interact with Artifactory.
*
*
* @author Oliver Gierke
* @author Mark Paluch
*/
@RequiredArgsConstructor
class ArtifactoryClient {
@@ -43,7 +44,7 @@ class ArtifactoryClient {
/**
* Triggers the promotion of the artifacts identified by the given {@link DeploymentInformation}.
*
*
* @param information must not be {@literal null}.
*/
public void promote(DeploymentInformation information) {
@@ -78,6 +79,7 @@ class ArtifactoryClient {
} catch (HttpClientErrorException o_O) {
handle("Authentication verification failed!", o_O);
throw new IllegalStateException("Authentication verification failed!");
}
}

19
release-tools/src/main/java/org/springframework/data/release/deployment/DeploymentOperations.java
View File

@@ -25,6 +25,7 @@ import org.springframework.util.Assert;
* Deployment functionality.
*
* @author Oliver Gierke
* @author Mark Paluch
*/
@Component
@RequiredArgsConstructor
@@ -54,4 +55,22 @@ public class DeploymentOperations {
client.promote(information);
}
/**
* Rolls back the given {@link DeploymentInformation}.
*
* @param information must not be {@literal null}.
*/
public void rollback(DeploymentInformation information) {
Assert.notNull(information, "DeploymentInformation must not be null!");
if (information.getModule().getIteration().isPublic()) {
logger.log(information.getModule(),
"Skipping build rollback as it's a public version and was deployed to Maven Central directly,");
return;
}
client.deleteArtifacts(information);
}
}

7
release-tools/src/main/java/org/springframework/data/release/model/Module.java
View File

@@ -21,9 +21,10 @@ import org.springframework.util.Assert;
/**
* @author Oliver Gierke
* @author Mark Paluch
*/
@Value
public class Module implements VersionAware, Comparable<Module> {
public class Module implements VersionAware, ProjectAware, Comparable<Module> {
private final Project project;
private final Version version;
@@ -59,7 +60,7 @@ public class Module implements VersionAware, Comparable<Module> {
/**
* Returns whether the current {@link Module} refers to the same project as the given one.
*
*
* @param module must not be {@literal null}.
* @return
*/
@@ -67,7 +68,7 @@ public class Module implements VersionAware, Comparable<Module> {
return this.project.equals(module.project);
}
/*
/*
* (non-Javadoc)
* @see java.lang.Comparable#compareTo(java.lang.Object)
*/

3
release-tools/src/main/java/org/springframework/data/release/model/ModuleIteration.java
View File

@@ -22,10 +22,11 @@ import lombok.RequiredArgsConstructor;
/**
* @author Oliver Gierke
* @author Mark Paluch
*/
@RequiredArgsConstructor(access = AccessLevel.PACKAGE)
@EqualsAndHashCode
public class ModuleIteration implements IterationVersion {
public class ModuleIteration implements IterationVersion, ProjectAware{
private final @Getter Module module;
private final @Getter TrainIteration trainIteration;

24
release-tools/src/main/java/org/springframework/data/release/model/ProjectAware.java Normal file
View File

@@ -0,0 +1,24 @@
/*
* Copyright 2019 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.data.release.model;
/**
* @author Mark Paluch
*/
public interface ProjectAware {
Project getProject();
}