Simplify JDK and Maven installation and version upgrading.

java-tools.properties now defines versions in a machine readable format so that we can install these versions into the Docker image verify those versions in our release tooling to ensure the environment matches what we expect.

Closes #32
This commit is contained in:
Mark Paluch
2023-02-20 11:55:08 +01:00
parent d01285fb75
commit 651a76ac07
15 changed files with 394 additions and 51 deletions

4
Jenkinsfile vendored
View File

@@ -25,7 +25,7 @@ pipeline {
steps {
script {
def image = docker.build("springci/spring-data-release-tools:0.7", "ci")
def image = docker.build("springci/spring-data-release-tools:0.8", "ci")
docker.withRegistry('', 'hub.docker.com-springbuildmaster') {
image.push()
}
@@ -39,7 +39,7 @@ pipeline {
}
agent {
docker {
image 'springci/spring-data-release-tools:0.7'
image 'springci/spring-data-release-tools:0.8'
}
}
options { timeout(time: 4, unit: 'HOURS') }

View File

@@ -17,19 +17,19 @@ RUN set -eux; \
sed -i -e 's/ports.ubuntu.com/ftp.tu-chemnitz.de\/pub\/linux/g' /etc/apt/sources.list && \
sed -i -e 's/http/https/g' /etc/apt/sources.list && \
apt-get update && \
apt-get -y install curl zip gnupg gnupg1 libfreetype6 fontconfig
RUN rm -rf /var/lib/apt/lists/* && \
apt-get -y install curl zip gnupg gnupg1 libfreetype6 fontconfig && \
rm -rf /var/lib/apt/lists/* && \
rm -rf /tmp/*
USER $USER_UID:$USER_GID
COPY java-init.sh /temp/java-init.sh
COPY java-tools.properties /temp/java-tools.properties
RUN curl -s "https://get.sdkman.io" | bash
RUN bash -c "source $HOME/.sdkman/bin/sdkman-init.sh && \
yes | sdk install java 17.0.6-tem && \
yes | sdk install java 8.0.352-tem && \
yes | sdk install java 8.0.362-tem && \
yes | sdk install maven && \
RUN bash -c "cd /temp && \
./java-init.sh && \
rm -rf $HOME/.sdkman/archives/* && \
rm -rf $HOME/.sdkman/tmp/*"

32
ci/java-init.sh Executable file
View File

@@ -0,0 +1,32 @@
#!/bin/bash
####################################################################
# Utility to install Java and Maven into the build container image #
####################################################################
source $HOME/.sdkman/bin/sdkman-init.sh
JAVA_TOOLS_PROPERTIES=java-tools.properties
if [ ! -f ${JAVA_TOOLS_PROPERTIES} ]
then
echo "File does not exist: ${JAVA_TOOLS_PROPERTIES}"
exit 1
fi
while IFS='=' read -r key value
do
key=$(echo $key | tr '.' '_')
eval ${key}=\${value}
done < "${JAVA_TOOLS_PROPERTIES}"
IFS=', ' read -r -a jdk_versions <<< "$jdks"
for to_install in "${jdk_versions[@]}"
do
dist="${to_install}-tem"
echo "Installing JDK ${dist}"
yes | sdk install java "${dist}"
done
echo "Installing Maven ${maven}"
yes | sdk install maven ${maven}

3
ci/java-tools.properties Normal file
View File

@@ -0,0 +1,3 @@
# Tool requirements
jdks=17.0.6,8.0.352,8.0.362
maven=3.9.0

View File

@@ -152,6 +152,10 @@ See `application-local.template` for details.
==== Utilities
===== Java and Maven Versions used in the Container
Java and Maven versions are installed via https://sdkman.io/[SDKman] during the link:ci/Dockerfile[`Dockerfile`] build. See link:ci/java-tools.properties[`ci/java-tools.properties`] for further details.
===== GitHub Labels
`ProjectLabelConfiguration` contains a per-project configuration which labels should be present in a project. To apply that configuration (create or update), use:

View File

@@ -22,7 +22,6 @@ import java.io.File;
import java.util.Map;
import org.apache.commons.io.FileUtils;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
import org.springframework.util.Assert;
@@ -37,7 +36,7 @@ import org.springframework.util.Assert;
@Data
@Component
@ConfigurationProperties(prefix = "maven")
class MavenProperties {
public class MavenProperties {
private File mavenHome;
private File localRepository;

View File

@@ -16,6 +16,7 @@
package org.springframework.data.release.build;
import lombok.RequiredArgsConstructor;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import java.io.Closeable;
@@ -24,7 +25,11 @@ import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.function.Consumer;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import org.apache.maven.shared.invoker.DefaultInvocationRequest;
@@ -32,7 +37,7 @@ import org.apache.maven.shared.invoker.DefaultInvoker;
import org.apache.maven.shared.invoker.InvocationRequest;
import org.apache.maven.shared.invoker.InvocationResult;
import org.apache.maven.shared.invoker.Invoker;
import org.apache.maven.shared.invoker.MavenInvocationException;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.data.release.io.JavaRuntimes;
import org.springframework.data.release.io.Workspace;
@@ -48,8 +53,9 @@ import org.springframework.stereotype.Component;
*/
@Slf4j
@Component
class MavenRuntime {
public class MavenRuntime {
private static final Pattern versionPattern = Pattern.compile("Apache Maven ((\\d\\.?)+) \\(.*\\)");
private final Workspace workspace;
private final Logger logger;
private final MavenProperties properties;
@@ -64,7 +70,7 @@ class MavenRuntime {
*/
@Autowired
public MavenRuntime(Workspace workspace, Logger logger, MavenProperties properties) {
this(workspace, logger, properties, JavaVersion.JAVA_8);
this(workspace, logger, properties, JavaVersion.VERSION_1_8);
}
private MavenRuntime(Workspace workspace, Logger logger, MavenProperties properties,
@@ -74,12 +80,37 @@ class MavenRuntime {
this.logger = logger;
this.properties = properties;
this.jdk = JavaRuntimes.Selector.from(requiredJavaVersion).notGraalVM().getRequiredJdkInstallation();
logger.log("Maven", "Using" + jdk + " as default Java Runtime");
}
public MavenRuntime withJavaVersion(JavaVersion javaVersion) {
return new MavenRuntime(workspace, logger, properties, javaVersion);
}
@SneakyThrows
public String getVersion() throws IllegalStateException {
StringBuilder builder = new StringBuilder();
Invoker invoker = new DefaultInvoker();
invoker.setMavenHome(properties.getMavenHome());
invoker.setErrorHandler(builder::append);
invoker.setOutputHandler(builder::append);
doWithMaven(invoker, mvn -> {
mvn.setShowVersion(true);
mvn.setGoals(Collections.emptyList());
});
Matcher matcher = versionPattern.matcher(builder);
boolean foundVersion = matcher.find();
if(!foundVersion){
throw new IllegalStateException("Cannot determine Maven Version: " + builder);
}
return matcher.group(1);
}
public MavenInvocationResult execute(Project project, CommandLine arguments) {
logger.log(project, "📦 Executing mvn %s", arguments.toString());
@@ -91,25 +122,15 @@ class MavenRuntime {
invoker.setOutputHandler(mavenLogger::info);
invoker.setErrorHandler(mavenLogger::warn);
File localRepository = properties.getLocalRepository();
InvocationResult result = doWithMaven(invoker, mvn -> {
if (localRepository != null) {
invoker.setLocalRepositoryDirectory(localRepository);
}
mvn.setBaseDirectory(workspace.getProjectDirectory(project));
mavenLogger.info(String.format("Java Home: %s", jdk));
mavenLogger.info(String.format("Executing: mvn %s", arguments));
File javaHome = getJavaHome();
mavenLogger.info(String.format("Java Home: %s", jdk));
mavenLogger.info(String.format("Executing: mvn %s", arguments));
mvn.setGoals(arguments.toCommandLine(it -> properties.getFullyQualifiedPlugin(it.getGoal())));
InvocationRequest request = new DefaultInvocationRequest();
request.setJavaHome(javaHome);
request.setShellEnvironmentInherited(true);
request.setBaseDirectory(workspace.getProjectDirectory(project));
request.setBatchMode(true);
request.setGoals(arguments.toCommandLine(it -> properties.getFullyQualifiedPlugin(it.getGoal())));
InvocationResult result = invoker.execute(request);
});
if (result.getExitCode() != 0) {
logger.warn(project, "🙈 Failed execution mvn %s", arguments.toString());
@@ -130,6 +151,26 @@ class MavenRuntime {
}
}
private InvocationResult doWithMaven(Invoker invoker, Consumer<InvocationRequest> mvn)
throws MavenInvocationException {
File localRepository = properties.getLocalRepository();
if (localRepository != null) {
invoker.setLocalRepositoryDirectory(localRepository);
}
File javaHome = getJavaHome();
InvocationRequest request = new DefaultInvocationRequest();
request.setJavaHome(javaHome);
request.setShellEnvironmentInherited(true);
request.setBatchMode(true);
mvn.accept(request);
return invoker.execute(request);
}
private File getJavaHome() {
return jdk.getHome().getAbsoluteFile();
}

View File

@@ -0,0 +1,39 @@
/*
* Copyright 2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.data.release.cli;
import lombok.Getter;
import java.io.File;
/**
* @author Mark Paluch
*/
@Getter
class InvalidMavenVersionException extends IllegalStateException {
private final String expectedVersion;
private final String actualVersion;
private final File home;
public InvalidMavenVersionException(String expectedVersion, String installedVersion, File home) {
super(String.format("Invalid Maven version: Expected %s, found version %s", expectedVersion, installedVersion));
this.expectedVersion = expectedVersion;
this.actualVersion = installedVersion;
this.home = home;
}
}

View File

@@ -0,0 +1,58 @@
/*
* Copyright 2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.data.release.cli;
import java.util.Properties;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.beans.factory.config.PropertiesFactoryBean;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.io.FileSystemResource;
import org.springframework.core.io.Resource;
import org.springframework.data.release.build.MavenProperties;
import org.springframework.data.release.build.MavenRuntime;
import org.springframework.data.release.utils.Logger;
/**
* Configuration to verify build infrastructure.
*
* @author Mark Paluch
*/
@Configuration
class JavaToolingConfiguration {
private static final Resource javaTools = new FileSystemResource("ci/java-tools.properties");
@Bean
PropertiesFactoryBean javaTools() {
if (!javaTools.exists()) {
throw new IllegalStateException(String.format("%s does not exist", javaTools));
}
PropertiesFactoryBean factory = new PropertiesFactoryBean();
factory.setLocations(javaTools);
return factory;
}
@Bean
JavaToolingVerifier verifier(@Qualifier("javaTools") Properties javaTools, MavenRuntime mavenRuntime,
MavenProperties mavenProperties, Logger logger) {
return new JavaToolingVerifier(javaTools, mavenRuntime, mavenProperties, logger);
}
}

View File

@@ -0,0 +1,93 @@
/*
* Copyright 2023 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.springframework.data.release.cli;
import lombok.RequiredArgsConstructor;
import java.util.Properties;
import javax.annotation.PostConstruct;
import org.springframework.boot.diagnostics.AbstractFailureAnalyzer;
import org.springframework.boot.diagnostics.FailureAnalysis;
import org.springframework.data.release.build.MavenProperties;
import org.springframework.data.release.build.MavenRuntime;
import org.springframework.data.release.io.JavaRuntimes.JdkInstallation;
import org.springframework.data.release.io.JavaRuntimes.Selector;
import org.springframework.data.release.model.JavaVersion;
import org.springframework.data.release.utils.Logger;
import org.springframework.util.StringUtils;
/**
* Utility to verify early on that your build environment contains all the required Java and Maven versions.
*
* @author Mark Paluch
*/
@RequiredArgsConstructor
class JavaToolingVerifier {
private final Properties javaTools;
private final MavenRuntime mavenRuntime;
private final MavenProperties mavenProperties;
private final Logger logger;
@PostConstruct
public void verify() {
String jdksProperty = javaTools.getProperty("jdks");
String[] jdks = jdksProperty.split(",");
logger.log("JavaTooling", "🕵️ Checking presence of JDKs %s…", StringUtils.arrayToDelimitedString(jdks, ", "));
for (String jdk : jdks) {
JavaVersion javaVersion = JavaVersion.of(jdk.trim());
JdkInstallation jdkInstallation = Selector.notGraalVM(javaVersion).getRequiredJdkInstallation();
logger.log("JavaTooling", "✅ Found %s by %s", javaVersion.getName(), jdkInstallation.getImplementor());
}
String expectedMavenVersion = javaTools.getProperty("maven");
logger.log("JavaTooling", "🕵️ Checking presence of Maven %s…", expectedMavenVersion);
String installedMavenVersion = mavenRuntime.getVersion();
if (!expectedMavenVersion.equals(installedMavenVersion)) {
throw new InvalidMavenVersionException(expectedMavenVersion, installedMavenVersion,
mavenProperties.getMavenHome());
}
logger.log("JavaTooling", "✅ Found Maven %s", installedMavenVersion);
}
static class InvalidMavenVersionExceptionFailureAnalyzer
extends AbstractFailureAnalyzer<InvalidMavenVersionException> {
@Override
protected FailureAnalysis analyze(Throwable rootFailure, InvalidMavenVersionException cause) {
return new FailureAnalysis(
String.format("⚠️ The configured Maven version %s at %s does not match the required version %s.",
cause.getActualVersion(), cause.getHome(), cause.getExpectedVersion()),
String.format(" Make sure to use Maven %s or update your maven.maven-home property.",
cause.getExpectedVersion()),
cause);
}
}
}

View File

@@ -34,7 +34,8 @@ import java.util.stream.Collectors;
import org.apache.commons.io.FileUtils;
import org.apache.commons.io.filefilter.RegexFileFilter;
import org.springframework.boot.diagnostics.AbstractFailureAnalyzer;
import org.springframework.boot.diagnostics.FailureAnalysis;
import org.springframework.boot.system.SystemProperties;
import org.springframework.data.release.model.JavaVersion;
import org.springframework.data.release.model.Version;
@@ -76,7 +77,7 @@ public class JavaRuntimes {
* @return
*/
public static JdkInstallation getJdk(Predicate<JdkInstallation> filter) {
return getJdk(filter, () -> "Cannot obtain required JDK");
return getJdk(filter, "Java Runtime", () -> "Cannot obtain required JDK");
}
/**
@@ -84,15 +85,18 @@ public class JavaRuntimes {
* first matching one or throws {@link NoSuchElementException}.
*
* @param filter
* @param runtimeName
* @param message
* @return
*/
public static JdkInstallation getJdk(Predicate<JdkInstallation> filter, Supplier<String> message) {
public static JdkInstallation getJdk(Predicate<JdkInstallation> filter, String runtimeName,
Supplier<String> message) {
List<JdkInstallation> jdks = JDKS.get();
return jdks.stream().filter(filter).findFirst()
.orElseThrow(() -> new NoSuchElementException(String.format("%s%nAvailable JDK: %s", message.get(), jdks)));
.orElseThrow(() -> new NoSuchJavaRuntimeException(String.format("%s%nAvailable JDK: %s", message.get(), jdks),
jdks, runtimeName));
}
public static List<JdkInstallation> getJdks() {
@@ -126,6 +130,8 @@ public class JavaRuntimes {
public static class Selector {
private String notFoundMessage;
private String javaRuntimeName;
private Predicate<JdkInstallation> predicate;
private Selector() {
@@ -141,7 +147,11 @@ public class JavaRuntimes {
return builder()
.and(it -> javaVersion.getVersionDetector().test(it.getVersion())
&& javaVersion.getImplementor().test(it.getImplementor()))
.message("Cannot find Java " + javaVersion.getName());
.name(javaVersion.getName()).message("Cannot find required " + javaVersion.getName());
}
public static Selector notGraalVM(JavaVersion javaVersion) {
return from(javaVersion).notGraalVM();
}
public Selector and(Predicate<JdkInstallation> predicate) {
@@ -159,8 +169,13 @@ public class JavaRuntimes {
return this;
}
public Selector name(String javaRuntimeName) {
this.javaRuntimeName = javaRuntimeName;
return this;
}
public JdkInstallation getRequiredJdkInstallation() {
return JavaRuntimes.getJdk(predicate, () -> notFoundMessage);
return JavaRuntimes.getJdk(predicate, javaRuntimeName, () -> notFoundMessage);
}
}
@@ -209,17 +224,24 @@ public class JavaRuntimes {
@SneakyThrows
private String parseImplementor(File candidateHome) {
List<String> release = FileUtils.readLines(new File(candidateHome, "release"));
File releaseMeta = new File(candidateHome, "release");
if (releaseMeta.exists()) {
List<String> release = FileUtils.readLines(releaseMeta);
for (String line : release) {
for (String line : release) {
if (line.startsWith("IMPLEMENTOR=")) {
String substring = line.substring(line.indexOf("=\""));
substring = substring.substring(2, substring.length() - 1);
return substring;
if (line.startsWith("IMPLEMENTOR=")) {
String substring = line.substring(line.indexOf("=\""));
substring = substring.substring(2, substring.length() - 1);
return substring;
}
}
}
if (candidateHome.getName().endsWith("-zulu")) {
return "Azul Systems, Inc.";
}
return "?";
}
}
@@ -340,4 +362,46 @@ public class JavaRuntimes {
return implementor + " " + version;
}
public static class NoSuchJavaRuntimeException extends NoSuchElementException {
private final List<JdkInstallation> installations;
private final String requiredJdk;
public NoSuchJavaRuntimeException(String message, List<JdkInstallation> installations, String requiredJdk) {
super(message);
this.installations = installations;
this.requiredJdk = requiredJdk;
}
public List<JdkInstallation> getInstallations() {
return installations;
}
public String getRequiredJdk() {
return requiredJdk;
}
}
static class NoSuchJavaRuntimeExceptionFailureAnalyzer extends AbstractFailureAnalyzer<NoSuchJavaRuntimeException> {
@Override
protected FailureAnalysis analyze(Throwable rootFailure, NoSuchJavaRuntimeException cause) {
String action = " Make sure to install %s using your platform installation method or SDKman.%n%n"
+ " Detected Java Runtimes are: %n" + "%s";
StringBuilder detectedRuntimes = new StringBuilder();
for (JdkInstallation installation : cause.getInstallations()) {
detectedRuntimes.append(String.format(" - %-20s %-10s %s%n", installation.getImplementor(),
installation.getVersion(), installation.getHome()));
}
return new FailureAnalysis("⚠️ A required JDK was not found: " + cause.getRequiredJdk(),
String.format(action, cause.getRequiredJdk(), detectedRuntimes), cause);
}
}
}

View File

@@ -29,22 +29,30 @@ import java.util.regex.Pattern;
@Value(staticConstructor = "of")
public class JavaVersion {
public static final JavaVersion VERSION_1_8 = of("1.8");
private static final Pattern DOCKER_TAG_PATTERN = Pattern.compile("((:?\\d+(:?u\\d+)?(:?\\.\\d+)*)).*");
public static final JavaVersion JAVA_8 = of("1.8.0_362");
public static final JavaVersion JAVA_17 = of("Java 17", version -> version.getMajor() == 17, it -> true);
String name;
Predicate<Version> versionDetector;
Predicate<String> implementor;
public static JavaVersion of(String version) {
Version expectedVersion = parse(version);
return of("Java " + version, candidate -> candidate.is(expectedVersion), it -> true);
return of("JDK " + version, candidate -> {
if (expectedVersion.getBugfix() == 0 && expectedVersion.getBuild() == 0) {
return candidate.withBugfix(0).is(expectedVersion);
}
return candidate.is(expectedVersion);
}, it -> true);
}
public static Version parse(String version) {
if (version.startsWith("8.")) {
version = "1." + version;
}
return Version.parse(version.replace('_', '.'));
}

View File

@@ -35,7 +35,6 @@ import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.commons.io.IOUtils;
import org.springframework.data.util.Streamable;
import org.springframework.lang.Nullable;
import org.springframework.util.Assert;
@@ -61,7 +60,7 @@ public class Train implements Streamable<Module> {
}
public Train(String name, Collection<Module> modules) {
this(name, Modules.of(modules), null, Iterations.DEFAULT, false, JavaVersion.JAVA_8);
this(name, Modules.of(modules), null, Iterations.DEFAULT, false, JavaVersion.VERSION_1_8);
}
/*
@@ -267,8 +266,8 @@ public class Train implements Streamable<Module> {
@ToString
public static class Iterations implements Iterable<Iteration> {
public static Iterations DEFAULT = new Iterations(M1, M2, M3, M4, M5, M6, RC1, RC2, GA, SR1, SR2, SR3, SR4, SR5, SR6, SR7, SR8,
SR9, SR10, SR11, SR12, SR13, SR14, SR15, SR16, SR17, SR18, SR19, SR20, SR21, SR22, SR23, SR24);
public static Iterations DEFAULT = new Iterations(M1, M2, M3, M4, M5, M6, RC1, RC2, GA, SR1, SR2, SR3, SR4, SR5,
SR6, SR7, SR8, SR9, SR10, SR11, SR12, SR13, SR14, SR15, SR16, SR17, SR18, SR19, SR20, SR21, SR22, SR23, SR24);
private final List<Iteration> iterations;

View File

@@ -242,4 +242,5 @@ public class Version implements Comparable<Version> {
return StringUtils.collectionToDelimitedString(digits, ".");
}
}

View File

@@ -0,0 +1,2 @@
org.springframework.boot.diagnostics.FailureAnalyzer=org.springframework.data.release.io.JavaRuntimes$NoSuchJavaRuntimeExceptionFailureAnalyzer,\
org.springframework.data.release.cli.JavaToolingVerifier.InvalidMavenVersionExceptionFailureAnalyzer