From 25bc0ef400e005bd64c74cc06d4492aed9f2caf9 Mon Sep 17 00:00:00 2001 From: Mark Paluch Date: Fri, 6 Sep 2019 10:44:22 +0200 Subject: [PATCH] #124 - Polishing. Add additional logging. Skip staging repository drop on Sonatype failures (e.g. when running into timeouts on repository close). Fix Artifactory error parsing. --- release-tools/application-local.template | 1 + .../data/release/build/BuildExecutor.java | 117 ++++++++++++++++-- .../data/release/build/BuildOperations.java | 25 +++- .../data/release/build/MavenBuildSystem.java | 2 +- .../release/deployment/ArtifactoryClient.java | 31 ++--- .../deployment/DeploymentProperties.java | 2 +- .../data/release/model/ReleaseTrains.java | 2 +- 7 files changed, 143 insertions(+), 37 deletions(-) diff --git a/release-tools/application-local.template b/release-tools/application-local.template index c1022f3..065ebdf 100644 --- a/release-tools/application-local.template +++ b/release-tools/application-local.template @@ -10,6 +10,7 @@ maven.mavenHome= # Deployment deployment.repository-prefix= +# Must be always the encrypted password taken from the Artifactory GUI/Profile view deployment.password= deployment.api-key= diff --git a/release-tools/src/main/java/org/springframework/data/release/build/BuildExecutor.java b/release-tools/src/main/java/org/springframework/data/release/build/BuildExecutor.java index a865937..733e79b 100644 --- a/release-tools/src/main/java/org/springframework/data/release/build/BuildExecutor.java +++ b/release-tools/src/main/java/org/springframework/data/release/build/BuildExecutor.java @@ -25,21 +25,25 @@ import java.util.Set; import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutionException; import java.util.concurrent.ExecutorService; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; import java.util.function.BiFunction; import java.util.function.Supplier; -import java.util.stream.Collectors; +import java.util.stream.Collector; import javax.annotation.PreDestroy; import org.springframework.data.release.Streamable; import org.springframework.data.release.model.Project; import org.springframework.data.release.model.ProjectAware; +import org.springframework.data.release.utils.ListWrapperCollector; +import org.springframework.data.release.utils.Logger; import org.springframework.plugin.core.PluginRegistry; import org.springframework.stereotype.Component; import org.springframework.util.Assert; +import org.springframework.util.StringUtils; import com.google.common.util.concurrent.MoreExecutors; @@ -52,17 +56,20 @@ import com.google.common.util.concurrent.MoreExecutors; class BuildExecutor { private final @NonNull PluginRegistry buildSystems; + private final Logger logger; private final MavenProperties mavenProperties; private final ExecutorService executor; - public BuildExecutor(PluginRegistry buildSystems, MavenProperties mavenProperties) { + public BuildExecutor(PluginRegistry buildSystems, Logger logger, + MavenProperties mavenProperties) { this.buildSystems = buildSystems; + this.logger = logger; this.mavenProperties = mavenProperties; if (this.mavenProperties.isParllelize()) { int processors = Runtime.getRuntime().availableProcessors(); - int parallelity = Math.max(2, processors - 2); + int parallelity = Math.max(2, (processors / 2)); executor = new ThreadPoolExecutor(parallelity, parallelity, 10, TimeUnit.MINUTES, new ArrayBlockingQueue<>(256)); } else { executor = MoreExecutors.newDirectExecutorService(); @@ -82,7 +89,7 @@ class BuildExecutor { * @param function must not be {@literal null}. * @return */ - public List doWithBuildSystemOrdered(Streamable iteration, + public Summary doWithBuildSystemOrdered(Streamable iteration, BiFunction function) { return doWithBuildSystem(iteration, function, true); } @@ -95,12 +102,12 @@ class BuildExecutor { * @param function must not be {@literal null}. * @return */ - public List doWithBuildSystemAnyOrder(Streamable iteration, + public Summary doWithBuildSystemAnyOrder(Streamable iteration, BiFunction function) { return doWithBuildSystem(iteration, function, false); } - private List doWithBuildSystem(Streamable iteration, + private Summary doWithBuildSystem(Streamable iteration, BiFunction function, boolean considerDependencyOrder) { Map> results = new ConcurrentHashMap<>(); @@ -136,8 +143,20 @@ class BuildExecutor { } return iteration.stream()// - .map(module -> results.get(module.getProject()).join()) // - .collect(Collectors.toList()); + .map(module -> { + + CompletableFuture future = results.get(module.getProject()); + + try { + return new ExecutionResult(module.getProject(), future.get()); + } + + catch (InterruptedException | ExecutionException e) { + return new ExecutionResult(module.getProject(), e.getCause()); + } + + }) // + .collect(toSummaryCollector()); } private CompletableFuture run(M module, BiFunction function) { @@ -165,4 +184,86 @@ class BuildExecutor { return result; } + /** + * Returns a new collector to toSummaryCollector {@link ExecutionResult} as {@link Summary} using the {@link Stream} + * API. + * + * @return + */ + public static Collector, ?, Summary> toSummaryCollector() { + return ListWrapperCollector.collectInto(Summary::new); + } + + public static class ExecutionResult { + + private final Project project; + private final T result; + private final Throwable failure; + + public ExecutionResult(Project project, Throwable failure) { + this.project = project; + this.result = null; + this.failure = failure; + } + + public ExecutionResult(Project project, T result) { + this.project = project; + this.result = result; + this.failure = null; + } + + public T getResult() { + return result; + } + + @Override + public String toString() { + return String.format("%20s - %s", project.getName(), + isSuccessful() ? "Successful" : "Error: " + failure.getMessage()); + } + + public boolean isSuccessful() { + return this.failure == null; + } + } + + public static class Summary { + + private final List> executions; + + public Summary(List> executions) { + this.executions = executions; + + if (!isSuccessful()) { + throw new BuildFailed(this); + } + } + + public List> getExecutions() { + return executions; + } + + public boolean isSuccessful() { + return this.executions.stream().allMatch(ExecutionResult::isSuccessful); + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + + builder.append("Execution summary"); + builder.append("\n"); + builder.append(StringUtils.collectionToDelimitedString(executions, "\n")); + + return builder.toString(); + } + } + + static class BuildFailed extends RuntimeException { + + public BuildFailed(Summary summary) { + super(summary.toString()); + } + } + } diff --git a/release-tools/src/main/java/org/springframework/data/release/build/BuildOperations.java b/release-tools/src/main/java/org/springframework/data/release/build/BuildOperations.java index a1eab1f..254705a 100644 --- a/release-tools/src/main/java/org/springframework/data/release/build/BuildOperations.java +++ b/release-tools/src/main/java/org/springframework/data/release/build/BuildOperations.java @@ -22,13 +22,16 @@ import java.nio.file.Path; import java.util.List; import java.util.function.BiFunction; import java.util.function.Supplier; +import java.util.stream.Collectors; import org.springframework.data.release.deployment.DeploymentInformation; +import org.springframework.data.release.model.Module; import org.springframework.data.release.model.ModuleIteration; import org.springframework.data.release.model.Phase; import org.springframework.data.release.model.Project; import org.springframework.data.release.model.Train; import org.springframework.data.release.model.TrainIteration; +import org.springframework.data.release.utils.Logger; import org.springframework.plugin.core.PluginRegistry; import org.springframework.stereotype.Component; import org.springframework.util.Assert; @@ -44,6 +47,7 @@ import com.google.common.annotations.VisibleForTesting; public class BuildOperations { private final @NonNull PluginRegistry buildSystems; + private final @NonNull Logger logger; private final @NonNull MavenProperties properties; private final @NonNull BuildExecutor executor; @@ -61,8 +65,10 @@ public class BuildOperations { UpdateInformation updateInformation = UpdateInformation.of(iteration, phase); - executor.doWithBuildSystemOrdered(iteration, + BuildExecutor.Summary summary = executor.doWithBuildSystemOrdered(iteration, (system, it) -> system.updateProjectDescriptors(it, updateInformation)); + + logger.log(iteration, "Update Project Descriptors done: %s", summary); } /** @@ -86,7 +92,10 @@ public class BuildOperations { Assert.notNull(train, "Train must not be null!"); - executor.doWithBuildSystemAnyOrder(train, BuildSystem::triggerDistributionBuild); + BuildExecutor.Summary summary = executor.doWithBuildSystemAnyOrder(train, + BuildSystem::triggerDistributionBuild); + + logger.log(train, "Distribution build: %s", summary); } /** @@ -96,8 +105,13 @@ public class BuildOperations { * @return */ public List performRelease(TrainIteration iteration) { - return executor.doWithBuildSystemOrdered(iteration, + + BuildExecutor.Summary summary = executor.doWithBuildSystemOrdered(iteration, (buildSystem, moduleIteration) -> performRelease(moduleIteration)); + + logger.log(iteration, "Release: %s", summary); + + return summary.getExecutions().stream().map(BuildExecutor.ExecutionResult::getResult).collect(Collectors.toList()); } /** @@ -121,7 +135,10 @@ public class BuildOperations { Assert.notNull(iteration, "Train iteration must not be null!"); Assert.notNull(phase, "Phase must not be null!"); - executor.doWithBuildSystemOrdered(iteration, (system, module) -> system.prepareVersion(module, phase)); + BuildExecutor.Summary summary = executor.doWithBuildSystemOrdered(iteration, + (system, module) -> system.prepareVersion(module, phase)); + + logger.log(iteration, "Prepare versions: %s", summary); } /** diff --git a/release-tools/src/main/java/org/springframework/data/release/build/MavenBuildSystem.java b/release-tools/src/main/java/org/springframework/data/release/build/MavenBuildSystem.java index c4f305f..b97c00a 100644 --- a/release-tools/src/main/java/org/springframework/data/release/build/MavenBuildSystem.java +++ b/release-tools/src/main/java/org/springframework/data/release/build/MavenBuildSystem.java @@ -338,7 +338,7 @@ class MavenBuildSystem implements BuildSystem { SKIP_TESTS, // arg("gpg.executable").withValue(gpg.getExecutable()), // arg("gpg.keyname").withValue(gpg.getKeyname()), // - arg("gpg.password").withValue(gpg.getPassword())); + arg("gpg.password").withValue(gpg.getPassword()), arg("skipStagingRepositoryClose").withValue("true")); mvn.execute(module.getProject(), arguments); } diff --git a/release-tools/src/main/java/org/springframework/data/release/deployment/ArtifactoryClient.java b/release-tools/src/main/java/org/springframework/data/release/deployment/ArtifactoryClient.java index f80d02f..8fc96d6 100644 --- a/release-tools/src/main/java/org/springframework/data/release/deployment/ArtifactoryClient.java +++ b/release-tools/src/main/java/org/springframework/data/release/deployment/ArtifactoryClient.java @@ -20,6 +20,7 @@ import lombok.Value; import java.io.IOException; import java.net.URI; +import java.util.function.Consumer; import org.springframework.data.release.model.ModuleIteration; import org.springframework.data.release.utils.Logger; @@ -61,7 +62,7 @@ class ArtifactoryClient { template.postForEntity(uri, new PromotionRequest(information.getTargetRepository(), properties.getStagingRepository()), String.class); } catch (HttpClientErrorException o_O) { - handle("Promotion failed!", o_O, module); + handle(message -> logger.warn(information.getModule(), message), "Promotion failed!", o_O); } } @@ -78,38 +79,24 @@ class ArtifactoryClient { logger.log("Artifactory", "Authentication verified!"); } catch (HttpClientErrorException o_O) { - handle("Authentication verification failed!", o_O); + handle(message -> logger.log("Artifactory Client", message), "Authentication verification failed!", o_O); throw new IllegalStateException("Authentication verification failed!"); } } - void handle(String log, HttpClientErrorException o_O, ModuleIteration module) { + private void handle(Consumer logger, String message, HttpClientErrorException o_O) { try { - logger.log(module, log); + logger.accept(message); Errors errors = new ObjectMapper().readValue(o_O.getResponseBodyAsByteArray(), Errors.class); - errors.getErrors().forEach(error -> logger.log(module, error)); - errors.getMessages().forEach(message -> logger.log(module, message)); + errors.getErrors().forEach(logger); + errors.getMessages().forEach(logger); } catch (IOException e) { - throw new RuntimeException(e); - } - } - - private void handle(String log, HttpClientErrorException o_O) { - - try { - - logger.log("Artifactory Client", log); - - Errors errors = new ObjectMapper().readValue(o_O.getResponseBodyAsByteArray(), Errors.class); - errors.getErrors().forEach(error -> logger.log("Artifactory Client", error)); - errors.getMessages().forEach(message -> logger.log("Artifactory Client", message)); - - } catch (IOException e) { - throw new RuntimeException(e); + o_O.addSuppressed(e); + throw new RuntimeException(o_O.getResponseBodyAsString(), o_O); } } diff --git a/release-tools/src/main/java/org/springframework/data/release/deployment/DeploymentProperties.java b/release-tools/src/main/java/org/springframework/data/release/deployment/DeploymentProperties.java index a446c30..e73c02a 100644 --- a/release-tools/src/main/java/org/springframework/data/release/deployment/DeploymentProperties.java +++ b/release-tools/src/main/java/org/springframework/data/release/deployment/DeploymentProperties.java @@ -92,7 +92,7 @@ public class DeploymentProperties { private static final String PROMOTION_RESOURCE = "/api/build/promote/{buildName}/{buildNumber}"; private static final String DELETE_BUILD_RESOURCE = "/api/build/{buildName}?buildNumbers={buildNumber}&artifacts=1"; - private static final String VERIFICATION_RESOURCE = "/api/storage/test-libs-staging-local"; + private static final String VERIFICATION_RESOURCE = "/api/storage/temp-private-local"; private String uri; diff --git a/release-tools/src/main/java/org/springframework/data/release/model/ReleaseTrains.java b/release-tools/src/main/java/org/springframework/data/release/model/ReleaseTrains.java index e1b69af..0a923c6 100644 --- a/release-tools/src/main/java/org/springframework/data/release/model/ReleaseTrains.java +++ b/release-tools/src/main/java/org/springframework/data/release/model/ReleaseTrains.java @@ -55,7 +55,7 @@ public class ReleaseTrains { LOVELACE = KAY.next("Lovelace", Transition.MINOR, new Module(JDBC, "1.0"), new Module(SOLR, "4.0")); - MOORE = LOVELACE.next("Moore", Transition.MINOR).withIterations(new Iterations(M1, M2, M3, M4, RC1, RC2, GA)); + MOORE = LOVELACE.next("Moore", Transition.MINOR).withIterations(new Iterations(M1, M2, M3, M4, RC1, RC2, RC3, GA)); // Trains