Support doing releases from CI.

This commit is contained in:
Greg L. Turnquist
2022-07-20 14:45:24 -05:00
parent 2534caf7fd
commit b0ccb313aa
31 changed files with 606 additions and 108 deletions

View File

@@ -21,12 +21,7 @@ import lombok.SneakyThrows;
import java.io.File;
import java.io.FileInputStream;
import java.util.Arrays;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.Set;
import java.util.*;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutionException;
@@ -39,7 +34,6 @@ import java.util.stream.Collectors;
import javax.annotation.PreDestroy;
import org.apache.commons.io.IOUtils;
import org.springframework.data.release.infra.InfrastructureOperations;
import org.springframework.data.release.io.Workspace;
import org.springframework.data.release.model.JavaVersion;
@@ -106,6 +100,13 @@ class BuildExecutor {
skip.forEach(it -> results.put(it, CompletableFuture.completedFuture(null)));
for (M moduleIteration : iteration) {
if (skip.contains(moduleIteration.getProject())) {
continue;
}
}
for (M moduleIteration : iteration) {
if (skip.contains(moduleIteration.getProject())) {
@@ -134,6 +135,7 @@ class BuildExecutor {
}
CompletableFuture<T> result = run(moduleIteration, function);
results.put(moduleIteration.getProject(), result);
}

View File

@@ -25,15 +25,8 @@ import java.util.function.Supplier;
import java.util.stream.Collectors;
import org.assertj.core.util.VisibleForTesting;
import org.springframework.data.release.deployment.DeploymentInformation;
import org.springframework.data.release.model.Module;
import org.springframework.data.release.model.ModuleIteration;
import org.springframework.data.release.model.Phase;
import org.springframework.data.release.model.Project;
import org.springframework.data.release.model.Projects;
import org.springframework.data.release.model.Train;
import org.springframework.data.release.model.TrainIteration;
import org.springframework.data.release.model.*;
import org.springframework.data.release.utils.Logger;
import org.springframework.plugin.core.PluginRegistry;
import org.springframework.stereotype.Component;
@@ -179,6 +172,21 @@ public class BuildOperations {
return properties.getLocalRepository().toPath();
}
/**
* Opens a repository to stage artifacts for this {@link ModuleIteration}.
*
* @param module must not be {@literal null}.
*/
public void open() {
buildSystems.getRequiredPluginFor(Projects.BUILD) //
.withJavaVersion(executor.detectJavaVersion(Projects.BUILD)).open();
}
public void close() {
buildSystems.getRequiredPluginFor(Projects.BUILD) //
.withJavaVersion(executor.detectJavaVersion(Projects.BUILD)).close();
}
/**
* Builds the release for the given {@link ModuleIteration} and deploys it to the staging repository.
*
@@ -241,5 +249,4 @@ public class BuildOperations {
return function.apply(buildSystem.withJavaVersion(executor.detectJavaVersion(module.getProject())), module);
}
}

View File

@@ -16,11 +16,7 @@
package org.springframework.data.release.build;
import org.springframework.data.release.deployment.DeploymentInformation;
import org.springframework.data.release.model.JavaVersion;
import org.springframework.data.release.model.ModuleIteration;
import org.springframework.data.release.model.Phase;
import org.springframework.data.release.model.Project;
import org.springframework.data.release.model.ProjectAware;
import org.springframework.data.release.model.*;
import org.springframework.plugin.core.Plugin;
/**
@@ -48,6 +44,13 @@ interface BuildSystem extends Plugin<Project> {
*/
ModuleIteration prepareVersion(ModuleIteration module, Phase phase);
/**
* Open a remote repository for staging artifacts.
*/
void open();
void close();
/**
* Deploy artifacts for the given {@link ModuleIteration} and return the {@link DeploymentInformation}.
*

View File

@@ -20,11 +20,7 @@ import lombok.NonNull;
import lombok.RequiredArgsConstructor;
import lombok.Value;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Optional;
import java.util.*;
import java.util.function.BooleanSupplier;
import java.util.function.Function;
import java.util.stream.Collectors;
@@ -181,6 +177,10 @@ class CommandLine {
return Argument.of("-X");
}
public static Argument settingsXml(String path) {
return Argument.of("-s " + path);
}
public Argument withValue(Object value) {
return new Argument(name, ArgumentValue.of(value));
}

View File

@@ -16,21 +16,18 @@
package org.springframework.data.release.build;
import static org.springframework.data.release.build.CommandLine.Argument.*;
import static org.springframework.data.release.build.CommandLine.Goal.*;
import static org.springframework.data.release.model.Projects.*;
import static org.springframework.data.release.model.Projects.BOM;
import static org.springframework.data.release.model.Projects.BUILD;
import lombok.AccessLevel;
import lombok.RequiredArgsConstructor;
import lombok.experimental.FieldDefaults;
import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.StringWriter;
import java.io.*;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.List;
import java.util.function.Consumer;
import java.util.regex.Pattern;
@@ -40,8 +37,9 @@ import javax.xml.transform.dom.DOMSource;
import javax.xml.transform.stream.StreamResult;
import org.apache.commons.io.IOUtils;
import org.springframework.core.annotation.Order;
import org.springframework.core.env.Environment;
import org.springframework.core.env.Profiles;
import org.springframework.data.release.build.CommandLine.Argument;
import org.springframework.data.release.build.CommandLine.Goal;
import org.springframework.data.release.build.Pom.Artifact;
@@ -49,18 +47,10 @@ import org.springframework.data.release.deployment.DefaultDeploymentInformation;
import org.springframework.data.release.deployment.DeploymentInformation;
import org.springframework.data.release.deployment.DeploymentProperties;
import org.springframework.data.release.io.Workspace;
import org.springframework.data.release.model.ArtifactVersion;
import org.springframework.data.release.model.Gpg;
import org.springframework.data.release.model.JavaVersion;
import org.springframework.data.release.model.ModuleIteration;
import org.springframework.data.release.model.Phase;
import org.springframework.data.release.model.Project;
import org.springframework.data.release.model.ProjectAware;
import org.springframework.data.release.model.TrainIteration;
import org.springframework.data.release.model.*;
import org.springframework.data.release.utils.Logger;
import org.springframework.stereotype.Component;
import org.springframework.util.Assert;
import org.xmlbeam.ProjectionFactory;
import org.xmlbeam.XBProjector;
import org.xmlbeam.dom.DOMAccess;
@@ -69,6 +59,7 @@ import org.xmlbeam.io.XBStreamInput;
/**
* @author Oliver Gierke
* @author Mark Paluch
* @author Greg Turnquist
*/
@Component
@Order(100)
@@ -85,10 +76,17 @@ class MavenBuildSystem implements BuildSystem {
DeploymentProperties properties;
Gpg gpg;
Environment env;
static String stagingRepositoryId = null;
static final String REPO_OPENING_TAG = "<repository>";
static final String REPO_CLOSING_TAG = "</repository>";
@Override
public BuildSystem withJavaVersion(JavaVersion javaVersion) {
return new MavenBuildSystem(workspace, projectionFactory, logger, mvn.withJavaVersion(javaVersion), properties,
gpg);
return new MavenBuildSystem(workspace, projectionFactory, logger, mvn.withJavaVersion(javaVersion), properties, gpg,
env);
}
/*
@@ -245,7 +243,7 @@ class MavenBuildSystem implements BuildSystem {
Project project = module.getProject();
UpdateInformation information = UpdateInformation.of(module.getTrainIteration(), phase);
CommandLine goals = CommandLine.of(goal("versions:set"), goal("versions:commit"));
CommandLine goals = CommandLine.of(Goal.goal("versions:set"), Goal.goal("versions:commit"));
if (BOM.equals(project)) {
@@ -277,11 +275,64 @@ class MavenBuildSystem implements BuildSystem {
return module;
}
/*
* (non-Javadoc)
* @see org.springframework.data.release.build.BuildSystem#deploy(org.springframework.data.release.model.ModuleIteration)
/**
* Perform a {@literal nexus-staging:rc-open} and extract the stagingProfileId from the results.
*/
@Override
public void open() {
try {
CommandLine arguments = CommandLine.of(Goal.goal("nexus-staging:rc-open"), //
profile("central"), //
of("-s " + properties.getSettingsXml()), //
arg("stagingProfileId").withValue(properties.getMavenCentral().getStagingProfileId()), //
arg("openedRepositoryMessageFormat").withValue("'" + REPO_OPENING_TAG + "%s" + REPO_CLOSING_TAG + "'"));
mvn.execute(BUILD, arguments);
String rcOpenLogfile = "mvn-" + BUILD.getName() + "-nexus-staging.rc-open.log";
logger.log(BUILD, "Searching " + this.workspace.getLogsDirectory().getAbsolutePath() + " for " + rcOpenLogfile);
Path rcOpenLogfilePath = Paths.get(this.workspace.getLogsDirectory().getAbsolutePath(), rcOpenLogfile);
logger.log(BUILD, "The log file is at " + rcOpenLogfilePath.toAbsolutePath() + " and "
+ (rcOpenLogfilePath.toFile().exists() ? " it exists!" : " it does NOT exist!"));
List<String> rcOpenLogContents = Files.readAllLines(rcOpenLogfilePath);
stagingRepositoryId = rcOpenLogContents.stream() //
.filter(line -> line.contains(REPO_OPENING_TAG) && !line.contains("%s")) //
.reduce((first, second) -> second) // find the last entry, a.k.a. the most recent log line
.map(s -> s.substring( //
s.indexOf(REPO_OPENING_TAG) + REPO_OPENING_TAG.length(), //
s.indexOf(REPO_CLOSING_TAG))) //
.orElse("");
logger.log(BUILD, "We just grabbed the staging repository ID at " + stagingRepositoryId);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
/**
* Perform a {@literal nexus-staging:rc-close}.
*/
@Override
public void close() {
CommandLine arguments = CommandLine.of(Goal.goal("nexus-staging:rc-close"), //
profile("central"), //
of("-s " + properties.getSettingsXml()), //
arg("stagingRepositoryId").withValue(stagingRepositoryId));
mvn.execute(BUILD, arguments);
}
/*
* (non-Javadoc)
* @see org.springframework.data.release.build.BuildSystem#deploy(org.springframework.data.release.model.ModuleIteration)
*/
@Override
public DeploymentInformation deploy(ModuleIteration module) {
Assert.notNull(module, "Module must not be null!");
@@ -289,6 +340,7 @@ class MavenBuildSystem implements BuildSystem {
DeploymentInformation information = new DefaultDeploymentInformation(module, properties);
deployToArtifactory(module, information);
deployToMavenCentral(module);
return information;
@@ -342,8 +394,8 @@ class MavenBuildSystem implements BuildSystem {
profile("central"), //
SKIP_TESTS, //
arg("gpg.executable").withValue(gpg.getExecutable()), //
arg("gpg.keyname").withValue(gpg.getKeyname()), //
arg("gpg.password").withValue(gpg.getPassword()));
arg("gpg.passphrase").withValue(gpg.getPassphrase()), //
arg("gpg.secretKeyring").withValue(gpg.getSecretKeyring()));
mvn.execute(BUILD, arguments);
@@ -403,9 +455,13 @@ class MavenBuildSystem implements BuildSystem {
CommandLine arguments = CommandLine.of(Goal.CLEAN, Goal.DEPLOY, //
profile("ci,release,central"), //
SKIP_TESTS, //
settingsXml(properties.getSettingsXml()), //
arg("gpg.executable").withValue(gpg.getExecutable()), //
arg("gpg.keyname").withValue(gpg.getKeyname()), //
arg("gpg.password").withValue(gpg.getPassword()));
arg("gpg.passphrase").withValue(gpg.getPassphrase()), //
arg("stagingRepositoryId").withValue(stagingRepositoryId)) //
.conditionalAnd(arg("gpg.secretKeyring").withValue(gpg.getSecretKeyring()),
() -> env.acceptsProfiles(Profiles.of("jenkins")));
mvn.execute(module.getProject(), arguments);
}

View File

@@ -18,20 +18,11 @@ package org.springframework.data.release.build;
import lombok.RequiredArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import java.io.Closeable;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
import java.io.*;
import java.util.List;
import java.util.stream.Collectors;
import org.apache.maven.shared.invoker.DefaultInvocationRequest;
import org.apache.maven.shared.invoker.DefaultInvoker;
import org.apache.maven.shared.invoker.InvocationRequest;
import org.apache.maven.shared.invoker.InvocationResult;
import org.apache.maven.shared.invoker.Invoker;
import org.apache.maven.shared.invoker.*;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.release.io.JavaRuntimes;
import org.springframework.data.release.io.Workspace;
@@ -79,6 +70,10 @@ class MavenRuntime {
return new MavenRuntime(workspace, logger, properties, javaVersion);
}
public MavenProperties getProperties() {
return this.properties;
}
public void execute(Project project, CommandLine arguments) {
logger.log(project, "📦 Executing mvn %s", arguments.toString());

View File

@@ -15,7 +15,7 @@
*/
package org.springframework.data.release.cli;
import static org.springframework.data.release.model.Projects.*;
import static org.springframework.data.release.model.Projects.COMMONS;
import lombok.AccessLevel;
import lombok.NonNull;
@@ -31,14 +31,7 @@ import org.springframework.data.release.git.GitOperations;
import org.springframework.data.release.issues.IssueTrackerCommands;
import org.springframework.data.release.issues.github.GitHubCommands;
import org.springframework.data.release.misc.ReleaseOperations;
import org.springframework.data.release.model.ArtifactVersion;
import org.springframework.data.release.model.ModuleIteration;
import org.springframework.data.release.model.Phase;
import org.springframework.data.release.model.Project;
import org.springframework.data.release.model.Projects;
import org.springframework.data.release.model.ReleaseTrains;
import org.springframework.data.release.model.Train;
import org.springframework.data.release.model.TrainIteration;
import org.springframework.data.release.model.*;
import org.springframework.shell.core.annotation.CliCommand;
import org.springframework.shell.core.annotation.CliOption;
import org.springframework.util.Assert;
@@ -111,6 +104,22 @@ class ReleaseCommands extends TimedCommand {
git.commit(iteration, "Release version %s.");
}
@CliCommand(value = "repository open")
public void repositoryOpen(@CliOption(key = "", mandatory = true) TrainIteration iteration) {
if (iteration.getIteration().isPublic()) {
build.open();
}
}
@CliCommand(value = "repository close")
public void repositoryClose(@CliOption(key = "", mandatory = true) TrainIteration iteration) {
if (iteration.getIteration().isPublic()) {
build.close();
}
}
@CliCommand(value = "release build")
public void buildRelease(@CliOption(key = "", mandatory = true) TrainIteration iteration, //
@CliOption(key = "project", mandatory = false) String projectName) {

View File

@@ -62,8 +62,12 @@ public class DeploymentProperties {
*/
private String distributionRepository;
private String settingsXml;
private String repositoryPrefix = "";
private MavenCentral mavenCentral;
public String getStagingRepository() {
return repositoryPrefix.concat(stagingRepository);
}
@@ -117,4 +121,11 @@ public class DeploymentProperties {
}
}
@Data
public static class MavenCentral {
private String stagingProfileId;
}
}

View File

@@ -34,12 +34,8 @@ import java.util.stream.Stream;
import java.util.stream.StreamSupport;
import org.apache.commons.io.FileUtils;
import org.eclipse.jgit.api.AddCommand;
import org.eclipse.jgit.api.CheckoutCommand;
import org.eclipse.jgit.api.CommitCommand;
import org.eclipse.jgit.api.*;
import org.eclipse.jgit.api.CreateBranchCommand.SetupUpstreamMode;
import org.eclipse.jgit.api.Git;
import org.eclipse.jgit.api.LogCommand;
import org.eclipse.jgit.api.ResetCommand.ResetType;
import org.eclipse.jgit.api.errors.EmptyCommitException;
import org.eclipse.jgit.api.errors.RefNotFoundException;
@@ -51,29 +47,15 @@ import org.eclipse.jgit.lib.Repository;
import org.eclipse.jgit.revwalk.RevCommit;
import org.eclipse.jgit.revwalk.RevWalk;
import org.eclipse.jgit.storage.file.FileRepositoryBuilder;
import org.eclipse.jgit.transport.CredentialItem;
import org.eclipse.jgit.transport.*;
import org.eclipse.jgit.transport.CredentialItem.CharArrayType;
import org.eclipse.jgit.transport.CredentialItem.InformationalMessage;
import org.eclipse.jgit.transport.CredentialsProvider;
import org.eclipse.jgit.transport.RefSpec;
import org.eclipse.jgit.transport.TagOpt;
import org.eclipse.jgit.transport.URIish;
import org.springframework.data.release.io.Workspace;
import org.springframework.data.release.issues.IssueTracker;
import org.springframework.data.release.issues.Ticket;
import org.springframework.data.release.issues.TicketReference;
import org.springframework.data.release.issues.TicketStatus;
import org.springframework.data.release.model.ArtifactVersion;
import org.springframework.data.release.model.Gpg;
import org.springframework.data.release.model.Iteration;
import org.springframework.data.release.model.ModuleIteration;
import org.springframework.data.release.model.Project;
import org.springframework.data.release.model.ProjectAware;
import org.springframework.data.release.model.Projects;
import org.springframework.data.release.model.ReleaseTrains;
import org.springframework.data.release.model.Train;
import org.springframework.data.release.model.TrainIteration;
import org.springframework.data.release.model.*;
import org.springframework.data.release.utils.ExecutionUtils;
import org.springframework.data.release.utils.Logger;
import org.springframework.data.util.Pair;
@@ -710,7 +692,9 @@ public class GitOperations {
.setAll(all);
if (gpg.isGpgAvailable()) {
commitCommand.setSign(true).setSigningKey(gpg.getKeyname())
commitCommand //
.setSign(true) //
.setSigningKey(gpg.getKeyname()) //
.setCredentialsProvider(new GpgPassphraseProvider(gpg));
} else {
commitCommand.setSign(false);
@@ -1101,6 +1085,7 @@ public class GitOperations {
}
private boolean matchesKey(CredentialItem[] items) {
return Arrays.stream(items).filter(InformationalMessage.class::isInstance) //
.map(CredentialItem::getPromptText) //
.map(it -> it.toLowerCase(Locale.US)) //
@@ -1116,7 +1101,7 @@ public class GitOperations {
for (CredentialItem item : items) {
if (item instanceof CharArrayType) {
((CharArrayType) item).setValueNoCopy(gpg.getPassword().toString().toCharArray());
((CharArrayType) item).setValueNoCopy(gpg.getPassphrase().toCharArray());
return true;
}

View File

@@ -21,7 +21,6 @@ import lombok.extern.slf4j.Slf4j;
import java.io.File;
import org.apache.commons.io.FileUtils;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;

View File

@@ -30,11 +30,12 @@ import org.springframework.util.StringUtils;
@ConfigurationProperties(prefix = "gpg")
public class Gpg {
private String keyname, executable;
private Password password;
private String keyname;
private String executable;
private String passphrase;
private String secretKeyring;
public boolean isGpgAvailable() {
return this.password != null && StringUtils.hasText(keyname);
return this.passphrase != null && StringUtils.hasText(secretKeyring);
}
}

View File

@@ -78,7 +78,7 @@ public class ReleaseTrains {
.filterModules(module -> !module.getProject().equals(ENVERS))
.filterModules(module -> !module.getProject().equals(R2DBC))
.filterModules(module -> !module.getProject().equals(JDBC)) // filter "old" JDBC without R2DBC submodule
.withIterations(new Train.Iterations(M1, M2, M3, M4, M5, RC1, RC2, GA, SR1, SR2, SR3, SR4, SR5));
.withIterations(new Train.Iterations(M1, M2, M3, M4, M5, M6, RC1, RC2, GA, SR1, SR2, SR3, SR4, SR5));
// Trains

View File

@@ -267,7 +267,7 @@ public class Train implements Streamable<Module> {
@ToString
public static class Iterations implements Iterable<Iteration> {
public static Iterations DEFAULT = new Iterations(M1, M2, M3, RC1, RC2, GA, SR1, SR2, SR3, SR4, SR5, SR6, SR7, SR8,
public static Iterations DEFAULT = new Iterations(M1, M2, M3, M4, M5, M6, RC1, RC2, GA, SR1, SR2, SR3, SR4, SR5, SR6, SR7, SR8,
SR9, SR10, SR11, SR12, SR13, SR14, SR15, SR16, SR17, SR18, SR19, SR20, SR21, SR22, SR23, SR24);
private final List<Iteration> iterations;

View File

@@ -0,0 +1,30 @@
git.username=${GIT_USERNAME}
git.author=${GIT_AUTHOR}
git.email=${GIT_EMAIL}
git.password=${GIT_PASSWORD}
github.api.url=${GITHUB_API_URL}
deployment.username=${DEPLOYMENT_USERNAME}
deployment.password=${DEPLOYMENT_PASSWORD}
deployment.api-key=${DEPLOYMENT_API_KEY}
deployment.maven-central.stagingProfileId=${STAGING_PROFILE_ID}
deployment.settings-xml=${SETTINGS_XML}
jira.username=${JIRA_USERNAME}
jira.password=${JIRA_PASSWORD}
jira.url=${JIRA_URL}
gpg.keyname=9A2C7A98E457C53D
gpg.passphrase=${PASSPHRASE}
gpg.secretKeyring=${GNUPGHOME}/secring.gpg
gpg.executable=/usr/bin/gpg
sagan.key=
maven.home=${MAVEN_HOME}
maven.console-logger=false
maven.repo.local=${RELEASE_TOOLS_CACHE}
io.workDir=dist
io.logs=${LOGS_DIR}

View File

@@ -15,7 +15,7 @@ github.team=christophstrobl,gregturn,jxblum,mp911de,odrotbohm,schauder,meisterme
# GPG setup
gpg.executable=/usr/local/bin/gpg2
# gpg.keyname
# gpg.password
# gpg.passphrase
# JIRA
jira.api-url=https://jira.spring.io
# GitHub

View File

@@ -10,7 +10,7 @@ jira.username=dummy
jira.password=dummy
jira.api-url=http://localhost:8888
git.username=dummy
git.password=dummy
git.passphrase=dummy
git.email=dummy@dummy.com
git.author=dummy
github.api-url=http://localhost:8888