Spring/spring-data-build
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

#838 - Introduce Jenkins.

Browse Source
This commit is contained in:
Greg Turnquist
2019-06-26 10:16:25 -05:00
parent af501507cb
commit 7b877b0e7c
11 changed files with 234 additions and 3352 deletions
Download Patch File Download Diff File Expand all files Collapse all files

110
.mvn/wrapper/MavenWrapperDownloader.java vendored
View File

@@ -1,110 +0,0 @@
/*
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
https://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing,
software distributed under the License is distributed on an
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
*/
import java.net.*;
import java.io.*;
import java.nio.channels.*;
import java.util.Properties;
public class MavenWrapperDownloader {
/**
* Default URL to download the maven-wrapper.jar from, if no 'downloadUrl' is provided.
*/
private static final String DEFAULT_DOWNLOAD_URL =
"https://repo.maven.apache.org/maven2/io/takari/maven-wrapper/0.4.2/maven-wrapper-0.4.2.jar";
/**
* Path to the maven-wrapper.properties file, which might contain a downloadUrl property to
* use instead of the default one.
*/
private static final String MAVEN_WRAPPER_PROPERTIES_PATH =
".mvn/wrapper/maven-wrapper.properties";
/**
* Path where the maven-wrapper.jar will be saved to.
*/
private static final String MAVEN_WRAPPER_JAR_PATH =
".mvn/wrapper/maven-wrapper.jar";
/**
* Name of the property which should be used to override the default download url for the wrapper.
*/
private static final String PROPERTY_NAME_WRAPPER_URL = "wrapperUrl";
public static void main(String args[]) {
System.out.println("- Downloader started");
File baseDirectory = new File(args[0]);
System.out.println("- Using base directory: " + baseDirectory.getAbsolutePath());
// If the maven-wrapper.properties exists, read it and check if it contains a custom
// wrapperUrl parameter.
File mavenWrapperPropertyFile = new File(baseDirectory, MAVEN_WRAPPER_PROPERTIES_PATH);
String url = DEFAULT_DOWNLOAD_URL;
if(mavenWrapperPropertyFile.exists()) {
FileInputStream mavenWrapperPropertyFileInputStream = null;
try {
mavenWrapperPropertyFileInputStream = new FileInputStream(mavenWrapperPropertyFile);
Properties mavenWrapperProperties = new Properties();
mavenWrapperProperties.load(mavenWrapperPropertyFileInputStream);
url = mavenWrapperProperties.getProperty(PROPERTY_NAME_WRAPPER_URL, url);
} catch (IOException e) {
System.out.println("- ERROR loading '" + MAVEN_WRAPPER_PROPERTIES_PATH + "'");
} finally {
try {
if(mavenWrapperPropertyFileInputStream != null) {
mavenWrapperPropertyFileInputStream.close();
}
} catch (IOException e) {
// Ignore ...
}
}
}
System.out.println("- Downloading from: : " + url);
File outputFile = new File(baseDirectory.getAbsolutePath(), MAVEN_WRAPPER_JAR_PATH);
if(!outputFile.getParentFile().exists()) {
if(!outputFile.getParentFile().mkdirs()) {
System.out.println(
"- ERROR creating output direcrory '" + outputFile.getParentFile().getAbsolutePath() + "'");
}
}
System.out.println("- Downloading to: " + outputFile.getAbsolutePath());
try {
downloadFileFromURL(url, outputFile);
System.out.println("Done");
System.exit(0);
} catch (Throwable e) {
System.out.println("- Error downloading");
e.printStackTrace();
System.exit(1);
}
}
private static void downloadFileFromURL(String urlString, File destination) throws Exception {
URL website = new URL(urlString);
ReadableByteChannel rbc;
rbc = Channels.newChannel(website.openStream());
FileOutputStream fos = new FileOutputStream(destination);
fos.getChannel().transferFrom(rbc, 0, Long.MAX_VALUE);
fos.close();
rbc.close();
}
}

101
Jenkinsfile vendored Normal file
View File

@@ -0,0 +1,101 @@
pipeline {
agent none
triggers {
pollSCM 'H/10 * * * *'
cron '* H * * *'
}
options {
disableConcurrentBuilds()
}
stages {
stage("Test") {
parallel {
stage("test: baseline") {
agent {
docker {
image 'adoptopenjdk/openjdk8:latest'
args '-v $HOME/.m2:/tmp/spring-data-maven-repository'
}
}
options { timeout(time: 30, unit: 'MINUTES') }
steps {
sh 'MAVEN_OPTS="-Duser.name=jenkins -Duser.home=/tmp/spring-data-maven-repository" ./mvnw clean dependency:list test -Dsort -B'
}
}
}
}
stage('Release to artifactory') {
when {
branch 'issue/*'
}
agent {
docker {
image 'adoptopenjdk/openjdk8:latest'
args '-v $HOME/.m2:/tmp/spring-data-maven-repository'
}
}
options { timeout(time: 20, unit: 'MINUTES') }
environment {
ARTIFACTORY = credentials('02bd1690-b54f-4c9f-819d-a77cb7a9822c')
}
steps {
sh 'MAVEN_OPTS="-Duser.name=jenkins -Duser.home=/tmp/spring-data-maven-repository" ./mvnw -Pci,artifactory ' +
'-Dartifactory.server=https://repo.spring.io ' +
"-Dartifactory.username=${ARTIFACTORY_USR} " +
"-Dartifactory.password=${ARTIFACTORY_PSW} " +
"-Dartifactory.staging-repository=libs-snapshot-local " +
"-Dartifactory.build-name=spring-data-build " +
"-Dartifactory.build-number=${BUILD_NUMBER} " +
'-Dmaven.test.skip=true clean deploy -B'
}
}
stage('Release to artifactory with docs') {
when {
branch 'master'
}
agent {
docker {
image 'adoptopenjdk/openjdk8:latest'
args '-v $HOME/.m2:/tmp/spring-data-maven-repository'
}
}
options { timeout(time: 20, unit: 'MINUTES') }
environment {
ARTIFACTORY = credentials('02bd1690-b54f-4c9f-819d-a77cb7a9822c')
}
steps {
sh 'MAVEN_OPTS="-Duser.name=jenkins -Duser.home=/tmp/spring-data-maven-repository" ./mvnw -Pci,artifactory ' +
'-Dartifactory.server=https://repo.spring.io ' +
"-Dartifactory.username=${ARTIFACTORY_USR} " +
"-Dartifactory.password=${ARTIFACTORY_PSW} " +
"-Dartifactory.staging-repository=libs-snapshot-local " +
"-Dartifactory.build-name=spring-data-build " +
"-Dartifactory.build-number=${BUILD_NUMBER} " +
'-Dmaven.test.skip=true clean deploy -B'
}
}
}
post {
changed {
script {
slackSend(
color: (currentBuild.currentResult == 'SUCCESS') ? 'good' : 'danger',
channel: '#spring-data-dev',
message: "${currentBuild.fullDisplayName} - `${currentBuild.currentResult}`\n${env.BUILD_URL}")
emailext(
subject: "[${currentBuild.fullDisplayName}] ${currentBuild.currentResult}",
mimeType: 'text/html',
recipientProviders: [[$class: 'CulpritsRecipientProvider'], [$class: 'RequesterRecipientProvider']],
body: "<a href=\"${env.BUILD_URL}\">${currentBuild.fullDisplayName} is reported as ${currentBuild.currentResult}</a>")
}
}
}
}

132
README.adoc Normal file
View File

@@ -0,0 +1,132 @@
= Spring Data Build Infrastructure
This repository contains common infrastructure to be used by Spring Data modules that build with Maven. It consists of a _resources_ project that bundles up resources that are needed during the build CSS for reference documentation generation and JavaDoc. The second project is _parent_ that can be used as parent project to pre-configure core dependencies, properties, reference documentation generation and most important of all the appropriate distribution assembly.
The parent project can be eased for either a single-module Maven project or a multi-module one. Each of the setups requires a slightly different setup of the project.
image:https://jenkins.spring.io/buildStatus/icon?job=spring-data-build%2Fmaster&subject=Moore%20(master)[link=https://jenkins.spring.io/view/SpringData/job/spring-data-build/]
image:https://jenkins.spring.io/buildStatus/icon?job=spring-data-build%2F2.1.x&subject=Lovelace%20(2.1.x)[link=https://jenkins.spring.io/view/SpringData/job/spring-data-build/]
image:https://jenkins.spring.io/buildStatus/icon?job=spring-data-build%2F1.9.x&subject=Ingalls%20(1.9.x)[link=https://jenkins.spring.io/view/SpringData/job/spring-data-build/]
== Project setup
=== General setup
The parent project configures the following aspects of the project build:
Shared resources are pulled in from the `spring-data-build-resources` dependency (images, CSS, XSLTs for documentation generation). Renders reference documentation from Docbook file named `index.xml` within `src/docbkx`. In the `distribute` profile, two assemblies are generated: A ZIP to be uploaded to static.springsource.org (incl. javadoc (browsable), reference docs as described before) with the following content:
[source]
----
- changelog.txt
- license.txt
- notice.txt
- readme.txt
+ reference -> Docbook generated reference documentation
+ html
+ pdf
+ epub
+ api -> JavaDoc
----
A second ZIP is generated to be uploaded to static.springsource.org to contain all Spring namespace XSDs.
The following dependencies are pre-configured.
* Logging dependencies: SLF4j + Commons Logging bridge and Logback as test dependency
* Test dependencies: JUnit / Hamcrest / Mockito
* Dependency versions for commonly used dependencies
=== Single project setup
If the client project is a project consisting of a single project only all that needs to be done is declaring the parent project:
[source,xml]
----
<parent>
<groupId>org.springframework.data.build</groupId>
<artifactId>spring-data-parent</artifactId>
<version>${most-recent-release-version}</version>
</parent>
----
Be sure to adapt the version number to the latest release version. The second and already last step of the setup is to activate the assembly and wagon plugin in the build section:
[source,xml]
----
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>wagon-maven-plugin</artifactId>
</plugin>
----
As an example have a look at the build of https://github.com/spring-projects/spring-data-jpa[Spring Data JPA].
=== Multi project setup
A multi module setup requires slightly more setup and some structure being set up.
* The root `pom.xml` needs to configure the `project.type` property to `multi`.
* Docbook documentation sources need to be in the root project.
* The assembly needs to be build in a dedicated sub-module (e.g. `distribution`), declare the assembly plugin (see single project setup) in that submodule and reconfigure the `project.root` property in that module to `${basedir}/..`.
* Configure `${dist.id}` in the root project to the basic artifact id (e.g. `spring-data-mongodb`) as this will serve as file name for distribution artifacts, static resources etc. It will default to the artifact id and thus usually resolve to a `…-parent` if not configured properly.
As an example have a look at the build of https://github.com/spring-projects/spring-data-mongodb[Spring Data MongoDB].
== Build configuration
* Configure "Artifactory Maven 3" task
* Goals to execute `clean (dependency:tree) install -Pci`
* A nightly build can then use `clean (dependency:tree) deploy -Pdistribute` to publish static resources and reference documentation
== Additional build profiles
* `ci` - Packages the JavaDoc as JAR for distribution (needs to be active on the CI server to make sure we distribute JavaDoc as JAR).
* `distribute` - Creates Docbook documentation, assembles the distribution zip, etc.
* `milestone` - Configures the binary distribution to upload to the milestone S3 repository.
* `release` - Configures the binary distribution to upload to the release S3 repository.
* `spring4-next` - Configures the Spring version to be used to be the next 4.0.x snapshot version.
* `spring41` - Configures the Spring version to be used to be the latest 4.1.x release version.
* `spring41-next` - Configures the Spring version to be used to be the next 4.1.x release version.
* `querydsl-next` - Configures the Querydsl version to be used to be the next available snapshot version.
== Running CI tasks locally
Since this pipeline is purely Docker-based, it's easy to:
* Debug what went wrong on your local machine.
* Experiment against a new image before submitting your pull request.
All of these use cases are great reasons to essentially run what the CI server does on your local machine.
IMPORTANT: To do this you must have Docker installed on your machine.
1. `docker run -it --mount type=bind,source="$(pwd)",target=/spring-data-build-github adoptopenjdk/openjdk8:latest /bin/bash`
+
This will launch the Docker image and mount your source code at `spring-data-build-github`.
+
2. `cd spring-data-build-github`
+
Next, run your tests from inside the container:
+
3. `./mvnw clean dependency:list test -Dsort` (or whatever profile you need to test out)
Since the container is binding to your source, you can make edits from your IDE and continue to run build jobs.
If you need to test the `build.sh` script, do this:
1. `docker run -it --mount type=bind,source="$(pwd)",target=/spring-data-build-github adoptopenjdk/openjdk8:latest /bin/bash`
+
This will launch the Docker image and mount your source code at `spring-data-build-github`.
+
2. `cd spring-data-build-github`
+
Next, try to package everything up from inside the container:
+
3. `./mvnw -Pci,snapshot -Dmaven.test.skip=true clean package`
NOTE: Docker containers can eat up disk space fast! From time to time, run `docker system prune` to clean out old images.

88
README.md
View File

@@ -1,88 +0,0 @@
# Spring Data Build Infrastructure
This repository contains common infrastructure to be used by Spring Data modules that build with Maven. It consists of a *resources* project that bundles up resources that are needed during the build CSS for reference documentation generation and JavaDoc. The second project is *parent* that can be used as parent project to pre-configure core dependencies, properties, reference documentation generation and most important of all the appropriate distribution assembly.
The parent project can be eased for either a single-module Maven project or a multi-module one. Each of the setups requires a slightly different setup of the project.
## Project setup
### General setup
The parent project configures the following aspects of the project build:
Shared resources are pulled in from the `spring-data-build-resources` dependency (images, CSS, XSLTs for documentation generation). Renders reference documentation from Docbook file named `index.xml` within `src/docbkx`. In the `distribute` profile, two assemblies are generated: A ZIP to be uploaded to static.springsource.org (incl. javadoc (browsable), reference docs as described before) with the following content:
```
- changelog.txt
- license.txt
- notice.txt
- readme.txt
+ reference -> Docbook generated reference documentation
+ html
+ pdf
+ epub
+ api -> JavaDoc
```
A second ZIP is generated to be uploaded to static.springsource.org to contain all Spring namespace XSDs.
The following dependencies are pre-configured.
- Logging dependencies: SLF4j + Commons Logging bridge and Logback as test dependency
- Test dependencies: JUnit / Hamcrest / Mockito
- Dependency versions for commonly used dependencies
### Single project setup
If the client project is a project consisting of a single project only all that needs to be done is declaring the parent project:
```xml
<parent>
<groupId>org.springframework.data.build</groupId>
<artifactId>spring-data-parent</artifactId>
<version>${most-recent-release-version}</version>
</parent>
```
Be sure to adapt the version number to the latest release version. The second and already last step of the setup is to activate the assembly and wagon plugin in the build section:
```xml
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-assembly-plugin</artifactId>
</plugin>
<plugin>
<groupId>org.codehaus.mojo</groupId>
<artifactId>wagon-maven-plugin</artifactId>
</plugin>
```
As an example have a look at the build of [Spring Data JPA](https://github.com/spring-projects/spring-data-jpa).
### Multi project setup
A multi module setup requires slightly more setup and some structure being set up.
- The root `pom.xml` needs to configure the `project.type` property to `multi`.
- Docbook documentation sources need to be in the root project.
- The assembly needs to be build in a dedicated sub-module (e.g. `distribution`), declare the assembly plugin (see single project setup) in that submodule and reconfigure the `project.root` property in that module to `${basedir}/..`.
- Configure `${dist.id}` in the root project to the basic artifact id (e.g. `spring-data-mongodb`) as this will serve as file name for distribution artifacts, static resources etc. It will default to the artifact id and thus usually resolve to a `…-parent` if not configured properly.
As an example have a look at the build of [Spring Data MongoDB](https://github.com/spring-projects/spring-data-mongodb).
## Build configuration
- Configure "Artifactory Maven 3" task
- Goals to execute `clean (dependency:tree) install -Pci`
- A nightly build can then use `clean (dependency:tree) deploy -Pdistribute` to publish static resources and reference documentation
## Additional build profiles
- `ci` - Packages the JavaDoc as JAR for distribution (needs to be active on the CI server to make sure we distribute JavaDoc as JAR).
- `distribute` - Creates Docbook documentation, assembles the distribution zip, etc.
- `milestone` - Configures the binary distribution to upload to the milestone S3 repository.
- `release` - Configures the binary distribution to upload to the release S3 repository.
- `spring4-next` - Configures the Spring version to be used to be the next 4.0.x snapshot version.
- `spring41` - Configures the Spring version to be used to be the latest 4.1.x release version.
- `spring41-next` - Configures the Spring version to be used to be the next 4.1.x release version.
- `querydsl-next` - Configures the Querydsl version to be used to be the next available snapshot version.

78
ci/README.adoc
View File

@@ -1,78 +0,0 @@
== Spring Data CI
Spring Data uses Concourse as it's CI tool of choice. This provides support for:
* Pipeline against the `master` branch
* Pull requests
=== Creating a pipeline
Using the `fly` command, you can execute a series of commands to create multiple pipelines to manage everything. But
first, some critical credentials are needed.
Create a `credentials.yml` file like this:
[source,yml]
----
github-access-token: <your Personal Access Token from github>
docker-email: <your docker hub email address>
docker-username: <your docker hub username>
docker-password: <your docker hub password>
artifactory-username: <your artifactory username>
artifactory-password: <your artifactory encoded password>
----
WARNING: Do NOT check this file into source control! If you'll check, `credentials.yml` is listed in `.gitignore` to prevent this.
With this in place, run the following `fly` commands to create pipelines:
----
% fly -t spring-data sp -p spring-data -c ci/pipeline-master.yml -l credentials.yml
----
With this pipeline in place, you can now activate and expose it:
----
% fly -t spring-data unpause-pipeline -p spring-data
% fly -t spring-data expose-pipeline -p spring-data
----
=== Running CI tasks locally
Since Concourse is built on top of Docker, it's easy to:
* Debug what went wrong on your local machine.
* Test out a a tweak to your `test.sh` script before sending it out.
* Experiment against a new image before submitting your pull request.
All of these use cases are great reasons to essentially run what Concourse does on your local machine.
IMPORTANT: To do this you must have Docker installed on your machine.
1. `docker run -it --mount type=bind,source="$(pwd)",target=/spring-data-build-github openjdk:8-jdk /bin/bash`
+
This will launch the Docker image and mount your source code at `spring-data-build-github`.
+
Next, run the `test.sh` script from inside the container:
+
2. `PROFILE=none spring-data-build-github/ci/test.sh`
Since the container is binding to your source, you can make edits from your IDE and continue to run build jobs.
If you need to test the `build.sh` script, then do this:
1. `mkdir /tmp/spring-data-build-artifactory`
2. `docker run -it --mount type=bind,source="$(pwd)",target=/spring-data-build-github --mount type=bind,source="/tmp/spring-data-build-artifactory",target=/spring-data-build-artifactory openjdk:8-jdk /bin/bash`
+
This will launch the Docker image and mount your source code at `spring-data-build-github` and the temporary
artifactory output directory at `spring-data-build-artifactory`.
+
Next, run the `build.sh` script from inside the container:
+
3. `spring-data-build-github/ci/build.sh`
IMPORTANT: `build.sh` doesn't actually push to Artifactory so don't worry about accidentally deploying anything.
It just deploys to a local folder. That way, the `artifactory-resource` later in the pipeline can pick up these artifacts
and deliver them to artifactory.
NOTE: Docker containers can eat up disk space fast! From time to time, run `docker system prune` to clean out old images.

15
ci/build.sh
View File

@@ -1,15 +0,0 @@
#!/bin/bash
set -euo pipefail
[[ -d $PWD/maven && ! -d $HOME/.m2 ]] && ln -s $PWD/maven $HOME/.m2
spring_data_build_artifactory=$(pwd)/spring-data-build-artifactory
rm -rf $HOME/.m2/repository/org/springframework/data 2> /dev/null || :
cd spring-data-build-github
./mvnw deploy \
-Dmaven.test.skip=true \
-DaltDeploymentRepository=distribution::default::file://${spring_data_build_artifactory}

20
ci/build.yml
View File

@@ -1,20 +0,0 @@
---
platform: linux
image_resource:
type: docker-image
source:
repository: openjdk
tag: 8-jdk
inputs:
- name: spring-data-build-github
outputs:
- name: spring-data-build-artifactory
caches:
- path: maven
run:
path: spring-data-build-github/ci/build.sh

3010
ci/pipeline-master.yml
View File

File diff suppressed because it is too large Load Diff

11
ci/test.sh
View File

@@ -1,11 +0,0 @@
#!/bin/bash
set -euo pipefail
[[ -d $PWD/maven && ! -d $HOME/.m2 ]] && ln -s $PWD/maven $HOME/.m2
rm -rf $HOME/.m2/repository/org/springframework/data 2> /dev/null || :
cd spring-data-build-github
./mvnw clean dependency:list test -P${PROFILE} -Dsort

20
ci/test.yml
View File

@@ -1,20 +0,0 @@
---
platform: linux
image_resource:
type: docker-image
source:
repository: openjdk
tag: 8-jdk
inputs:
- name: spring-data-build-github
outputs:
- name: spring-data-build-artifactory
caches:
- path: maven
run:
path: spring-data-build-github/ci/test.sh

1
pom.xml
View File

@@ -150,6 +150,7 @@
<buildInfo>
<buildName>{{artifactory.build-name}}</buildName>
<buildNumber>{{artifactory.build-number}}</buildNumber>
<buildUrl>{{BUILD_URL}}</buildUrl>
</buildInfo>
</configuration>
</execution>