From e549168b6fd01fe180048496e8a47dbf21cce39f Mon Sep 17 00:00:00 2001 From: Stephane Nicoll Date: Sun, 27 Feb 2022 18:20:56 +0100 Subject: [PATCH] Move CI to Concourse --- ci/README.adoc | 18 +++ ci/config/changelog-generator.yml | 16 ++ ci/config/release-scripts.yml | 10 ++ ci/images/README.adoc | 21 +++ ci/images/ci-image/Dockerfile | 8 + ci/images/get-jdk-url.sh | 11 ++ ci/images/setup.sh | 28 ++++ ci/parameters.yml | 12 ++ ci/pipeline.yml | 245 ++++++++++++++++++++++++++++++ ci/scripts/build-project.sh | 13 ++ ci/scripts/common.sh | 4 + ci/scripts/generate-changelog.sh | 12 ++ ci/scripts/promote.sh | 18 +++ ci/scripts/stage.sh | 39 +++++ ci/tasks/build-ci-image.yml | 31 ++++ ci/tasks/build-pr-project.yml | 10 ++ ci/tasks/build-project.yml | 10 ++ ci/tasks/generate-changelog.yml | 24 +++ ci/tasks/promote.yml | 17 +++ ci/tasks/stage.yml | 11 ++ 20 files changed, 558 insertions(+) create mode 100644 ci/README.adoc create mode 100644 ci/config/changelog-generator.yml create mode 100644 ci/config/release-scripts.yml create mode 100644 ci/images/README.adoc create mode 100644 ci/images/ci-image/Dockerfile create mode 100755 ci/images/get-jdk-url.sh create mode 100755 ci/images/setup.sh create mode 100644 ci/parameters.yml create mode 100644 ci/pipeline.yml create mode 100755 ci/scripts/build-project.sh create mode 100644 ci/scripts/common.sh create mode 100755 ci/scripts/generate-changelog.sh create mode 100755 ci/scripts/promote.sh create mode 100755 ci/scripts/stage.sh create mode 100644 ci/tasks/build-ci-image.yml create mode 100644 ci/tasks/build-pr-project.yml create mode 100644 ci/tasks/build-project.yml create mode 100644 ci/tasks/generate-changelog.yml create mode 100644 ci/tasks/promote.yml create mode 100644 ci/tasks/stage.yml diff --git a/ci/README.adoc b/ci/README.adoc new file mode 100644 index 0000000..fb40a5f --- /dev/null +++ b/ci/README.adoc @@ -0,0 +1,18 @@ +== Concourse pipeline + +Ensure that you've setup the target and can login + +[source] +---- +$ fly -t spring-retry login -n spring-retry -c https://ci.spring.io +---- + +The pipeline can be deployed using the following command: + +[source] +---- +$ fly -t spring-retry set-pipeline -p spring-retry-1.3.x -c ci/pipeline.yml -l ci/parameters.yml +---- + +NOTE: This assumes that you have credhub integration configured with the appropriate +secrets. diff --git a/ci/config/changelog-generator.yml b/ci/config/changelog-generator.yml new file mode 100644 index 0000000..97cd469 --- /dev/null +++ b/ci/config/changelog-generator.yml @@ -0,0 +1,16 @@ +changelog: + repository: spring-projects/spring-retry + sections: + - title: ":star: New Features" + labels: + - "enhancement" + - title: ":lady_beetle: Bug Fixes" + labels: + - "bug" + - title: ":notebook_with_decorative_cover: Documentation" + labels: + - "documentation" + - title: ":hammer: Dependency Upgrades" + sort: "title" + labels: + - "dependencies" diff --git a/ci/config/release-scripts.yml b/ci/config/release-scripts.yml new file mode 100644 index 0000000..d31f8cb --- /dev/null +++ b/ci/config/release-scripts.yml @@ -0,0 +1,10 @@ +logging: + level: + io.spring.concourse: DEBUG +spring: + main: + banner-mode: off +sonatype: + exclude: + - 'build-info\.json' + - '.*\.zip' diff --git a/ci/images/README.adoc b/ci/images/README.adoc new file mode 100644 index 0000000..92fa3c2 --- /dev/null +++ b/ci/images/README.adoc @@ -0,0 +1,21 @@ +== CI Images + +These images are used by CI to run the actual builds. + +To build the image locally run the following from this directory: + +---- +$ docker build --no-cache -f /Dockerfile . +---- + +For example + +---- +$ docker build --no-cache -f ci-image/Dockerfile . +---- + +To test run: + +---- +$ docker run -it --entrypoint /bin/bash +---- diff --git a/ci/images/ci-image/Dockerfile b/ci/images/ci-image/Dockerfile new file mode 100644 index 0000000..2ddd3ea --- /dev/null +++ b/ci/images/ci-image/Dockerfile @@ -0,0 +1,8 @@ +FROM ubuntu:focal-20220113 + +ADD setup.sh /setup.sh +ADD get-jdk-url.sh /get-jdk-url.sh +RUN ./setup.sh java8 + +ENV JAVA_HOME /opt/openjdk +ENV PATH $JAVA_HOME/bin:$PATH \ No newline at end of file diff --git a/ci/images/get-jdk-url.sh b/ci/images/get-jdk-url.sh new file mode 100755 index 0000000..f71d8e1 --- /dev/null +++ b/ci/images/get-jdk-url.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -e + +case "$1" in + java8) + echo "https://github.com/adoptium/temurin8-binaries/releases/download/jdk8u322-b06/OpenJDK8U-jdk_x64_linux_hotspot_8u322b06.tar.gz" + ;; + *) + echo $"Unknown java version" + exit 1 +esac diff --git a/ci/images/setup.sh b/ci/images/setup.sh new file mode 100755 index 0000000..32a332c --- /dev/null +++ b/ci/images/setup.sh @@ -0,0 +1,28 @@ +#!/bin/bash +set -ex + +########################################################### +# UTILS +########################################################### + +export DEBIAN_FRONTEND=noninteractive +apt-get update +apt-get install --no-install-recommends -y tzdata ca-certificates net-tools libxml2-utils git curl libudev1 libxml2-utils iptables iproute2 jq unzip +ln -fs /usr/share/zoneinfo/UTC /etc/localtime +dpkg-reconfigure --frontend noninteractive tzdata +rm -rf /var/lib/apt/lists/* + +curl https://raw.githubusercontent.com/spring-io/concourse-java-scripts/v0.0.4/concourse-java.sh > /opt/concourse-java.sh + +curl --output /opt/concourse-release-scripts.jar https://repo.spring.io/release/io/spring/concourse/releasescripts/concourse-release-scripts/0.3.2/concourse-release-scripts-0.3.2.jar + +########################################################### +# JAVA +########################################################### +JDK_URL=$( ./get-jdk-url.sh $1 ) + +mkdir -p /opt/openjdk +cd /opt/openjdk +curl -L ${JDK_URL} | tar zx --strip-components=1 +test -f /opt/openjdk/bin/java +test -f /opt/openjdk/bin/javac \ No newline at end of file diff --git a/ci/parameters.yml b/ci/parameters.yml new file mode 100644 index 0000000..26573dc --- /dev/null +++ b/ci/parameters.yml @@ -0,0 +1,12 @@ +github-repo: "https://github.com/spring-projects/spring-retry.git" +github-repo-name: "spring-projects/spring-retry" +docker-hub-organization: "springci" +artifactory-server: "https://repo.spring.io" +branch: "main" +milestone: "1.3.x" +build-name: "spring-retry" +concourse-url: "https://ci.spring.io" +task-timeout: 30m +registry-mirror-host: docker.repo.spring.io +registry-mirror-username: ((artifactory-username)) +registry-mirror-password: ((artifactory-password)) \ No newline at end of file diff --git a/ci/pipeline.yml b/ci/pipeline.yml new file mode 100644 index 0000000..6a191d0 --- /dev/null +++ b/ci/pipeline.yml @@ -0,0 +1,245 @@ +anchors: + artifactory-repo-put-params: &artifactory-repo-put-params + signing_key: ((signing-key)) + signing_passphrase: ((signing-passphrase)) + repo: libs-snapshot-local + folder: distribution-repository + build_uri: "https://ci.spring.io/teams/${BUILD_TEAM_NAME}/pipelines/${BUILD_PIPELINE_NAME}/jobs/${BUILD_JOB_NAME}/builds/${BUILD_NAME}" + build_number: "${BUILD_PIPELINE_NAME}-${BUILD_JOB_NAME}-${BUILD_NAME}" + disable_checksum_uploads: true + artifactory-task-params: &artifactory-task-params + ARTIFACTORY_SERVER: ((artifactory-server)) + ARTIFACTORY_USERNAME: ((artifactory-username)) + ARTIFACTORY_PASSWORD: ((artifactory-password)) + build-project-task-params: &build-project-task-params + privileged: true + timeout: ((task-timeout)) + file: git-repo/ci/tasks/build-project.yml + git-repo-resource-source: &git-repo-resource-source + uri: ((github-repo)) + username: ((github-username)) + password: ((github-ci-release-token)) + branch: ((branch)) + registry-image-resource-source: ®istry-image-resource-source + username: ((docker-hub-username)) + password: ((docker-hub-password)) + tag: ((milestone)) + sontatype-task-params: &sonatype-task-params + SONATYPE_USERNAME: ((sonatype-username)) + SONATYPE_PASSWORD: ((sonatype-password)) + SONATYPE_URL: ((sonatype-url)) + SONATYPE_STAGING_PROFILE_ID: ((sonatype-staging-profile-id)) + registry-mirror-vars: ®istry-mirror-vars + registry-mirror-host: ((registry-mirror-host)) + registry-mirror-username: ((registry-mirror-username)) + registry-mirror-password: ((registry-mirror-password)) +resource_types: +- name: artifactory-resource + type: registry-image + source: + repository: springio/artifactory-resource + tag: 0.0.14 +- name: pull-request + type: registry-image + source: + repository: teliaoss/github-pr-resource + tag: v0.23.0 +- name: github-release + type: registry-image + source: + repository: concourse/github-release-resource + tag: 1.5.5 +- name: github-status-resource + type: registry-image + source: + repository: dpb587/github-status-resource + tag: master +resources: +- name: git-repo + type: git + icon: github + source: + <<: *git-repo-resource-source +- name: git-pull-request + type: pull-request + icon: source-pull + source: + access_token: ((github-ci-pull-request-token)) + repository: ((github-repo-name)) + base_branch: ((branch)) + ignore_paths: ["ci/*"] +- name: github-release + type: github-release + icon: briefcase-download + source: + owner: spring-projects + repository: spring-retry + access_token: ((github-ci-release-token)) + pre_release: false +- name: ci-images-git-repo + type: git + icon: github + source: + uri: ((github-repo)) + branch: ((branch)) + paths: ["ci/images/*"] +- name: ci-image + type: registry-image + icon: docker + source: + <<: *registry-image-resource-source + repository: ((docker-hub-organization))/spring-retry-ci +- name: artifactory-repo + type: artifactory-resource + icon: package-variant + source: + uri: ((artifactory-server)) + username: ((artifactory-username)) + password: ((artifactory-password)) + build_name: ((build-name)) +- name: repo-status-build + type: github-status-resource + icon: eye-check-outline + source: + repository: ((github-repo-name)) + access_token: ((github-ci-status-token)) + branch: ((branch)) + context: build +jobs: +- name: build-ci-image + plan: + - get: ci-images-git-repo + trigger: true + - get: git-repo + - in_parallel: + - task: build-ci-image + privileged: true + file: git-repo/ci/tasks/build-ci-image.yml + output_mapping: + image: ci-image + vars: + ci-image-name: ci-image + <<: *registry-mirror-vars + - in_parallel: + - put: ci-image + params: + image: ci-image/image.tar +- name: build + serial: true + public: true + plan: + - get: ci-image + - get: git-repo + trigger: true + - put: repo-status-build + params: { state: "pending", commit: "git-repo" } + - do: + - task: build-project + image: ci-image + <<: *build-project-task-params + on_failure: + do: + - put: repo-status-build + params: { state: "failure", commit: "git-repo" } + - put: repo-status-build + params: { state: "success", commit: "git-repo" } + - put: artifactory-repo + params: + <<: *artifactory-repo-put-params +- name: build-pull-requests + serial: true + public: true + plan: + - get: ci-image + - get: git-repo + resource: git-pull-request + trigger: true + version: every + - do: + - put: git-pull-request + params: + path: git-repo + status: pending + - task: build-project + image: ci-image + file: git-repo/ci/tasks/build-pr-project.yml + on_success: + put: git-pull-request + params: + path: git-repo + status: success + on_failure: + put: git-pull-request + params: + path: git-repo + status: failure +- name: stage-release + serial: true + plan: + - get: ci-image + - get: git-repo + trigger: false + - task: stage + image: ci-image + file: git-repo/ci/tasks/stage.yml + params: + RELEASE_TYPE: RELEASE + - put: artifactory-repo + params: + <<: *artifactory-repo-put-params + repo: libs-staging-local + - put: git-repo + params: + repository: stage-git-repo +- name: promote-release + serial: true + plan: + - get: ci-image + - get: git-repo + trigger: false + - get: artifactory-repo + trigger: false + passed: [stage-release] + params: + download_artifacts: true + save_build_info: true + - task: promote + image: ci-image + file: git-repo/ci/tasks/promote.yml + params: + RELEASE_TYPE: RELEASE + <<: *artifactory-task-params + <<: *sonatype-task-params +- name: create-github-release + serial: true + plan: + - get: ci-image + - get: git-repo + - get: artifactory-repo + trigger: true + passed: [promote-release] + params: + download_artifacts: false + save_build_info: true + - task: generate-changelog + file: git-repo/ci/tasks/generate-changelog.yml + params: + RELEASE_TYPE: RELEASE + GITHUB_USERNAME: ((github-username)) + GITHUB_TOKEN: ((github-ci-release-token)) + vars: + <<: *registry-mirror-vars + - put: github-release + params: + name: generated-changelog/tag + tag: generated-changelog/tag + body: generated-changelog/changelog.md +groups: +- name: "build" + jobs: ["build"] +- name: "releases" + jobs: ["stage-release", "promote-release", "create-github-release"] +- name: "ci-images" + jobs: ["build-ci-image"] +- name: "pull-requests" + jobs: ["build-pull-requests"] diff --git a/ci/scripts/build-project.sh b/ci/scripts/build-project.sh new file mode 100755 index 0000000..2fbb4de --- /dev/null +++ b/ci/scripts/build-project.sh @@ -0,0 +1,13 @@ +#!/bin/bash +set -e + +source $(dirname $0)/common.sh +repository=$(pwd)/distribution-repository +if [ -z "$(ls -A maven)" ]; then + echo "Maven cache not available." +else + echo "Maven cache found." +fi +pushd git-repo > /dev/null +./mvnw clean deploy -U -Pfull -DaltDeploymentRepository=distribution::default::file://${repository} +popd > /dev/null diff --git a/ci/scripts/common.sh b/ci/scripts/common.sh new file mode 100644 index 0000000..cce6a16 --- /dev/null +++ b/ci/scripts/common.sh @@ -0,0 +1,4 @@ +source /opt/concourse-java.sh + +setup_symlinks +cleanup_maven_repo "org.springframework.retry" diff --git a/ci/scripts/generate-changelog.sh b/ci/scripts/generate-changelog.sh new file mode 100755 index 0000000..d3d2b97 --- /dev/null +++ b/ci/scripts/generate-changelog.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -e + +CONFIG_DIR=git-repo/ci/config +version=$( cat artifactory-repo/build-info.json | jq -r '.buildInfo.modules[0].id' | sed 's/.*:.*:\(.*\)/\1/' ) + +java -jar /github-changelog-generator.jar \ + --spring.config.location=${CONFIG_DIR}/changelog-generator.yml \ + ${version} generated-changelog/changelog.md + +echo ${version} > generated-changelog/version +echo v${version} > generated-changelog/tag diff --git a/ci/scripts/promote.sh b/ci/scripts/promote.sh new file mode 100755 index 0000000..44c5ff6 --- /dev/null +++ b/ci/scripts/promote.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +source $(dirname $0)/common.sh +CONFIG_DIR=git-repo/ci/config + +version=$( cat artifactory-repo/build-info.json | jq -r '.buildInfo.modules[0].id' | sed 's/.*:.*:\(.*\)/\1/' ) +export BUILD_INFO_LOCATION=$(pwd)/artifactory-repo/build-info.json + +java -jar /opt/concourse-release-scripts.jar \ + --spring.config.location=${CONFIG_DIR}/release-scripts.yml \ + publishToCentral $RELEASE_TYPE $BUILD_INFO_LOCATION artifactory-repo || { exit 1; } + +java -jar /opt/concourse-release-scripts.jar \ + --spring.config.location=${CONFIG_DIR}/release-scripts.yml \ + promote $RELEASE_TYPE $BUILD_INFO_LOCATION || { exit 1; } + +echo "Promotion complete" +echo $version > version/version diff --git a/ci/scripts/stage.sh b/ci/scripts/stage.sh new file mode 100755 index 0000000..0e7adbd --- /dev/null +++ b/ci/scripts/stage.sh @@ -0,0 +1,39 @@ +#!/bin/bash +set -e + +source $(dirname $0)/common.sh +repository=$(pwd)/distribution-repository + +pushd git-repo > /dev/null +git fetch --tags --all > /dev/null +popd > /dev/null + +git clone git-repo stage-git-repo > /dev/null + +pushd stage-git-repo > /dev/null + +snapshotVersion=$( get_revision_from_pom ) +stageVersion=$( get_next_release $snapshotVersion) +nextVersion=$( bump_version_number $snapshotVersion) +echo "Staging $stageVersion (next version will be $nextVersion)" + +set_revision_to_pom "$stageVersion" +git config user.name "Spring Builds" > /dev/null +git config user.email "spring-builds@users.noreply.github.com" > /dev/null +git add pom.xml > /dev/null +git commit -m"Release v$stageVersion" > /dev/null +git tag -a "v$stageVersion" -m"Release v$stageVersion" > /dev/null + +./mvnw clean deploy -U -Pfull -DaltDeploymentRepository=distribution::default::file://${repository} + +git reset --hard HEAD^ > /dev/null +if [[ $nextVersion != $snapshotVersion ]]; then + echo "Setting next development version (v$nextVersion)" + set_revision_to_pom "$nextVersion" + git add pom.xml > /dev/null + git commit -m"Next development version (v$nextVersion)" > /dev/null +fi; + +echo "DONE" + +popd > /dev/null diff --git a/ci/tasks/build-ci-image.yml b/ci/tasks/build-ci-image.yml new file mode 100644 index 0000000..25228e8 --- /dev/null +++ b/ci/tasks/build-ci-image.yml @@ -0,0 +1,31 @@ +--- +platform: linux +image_resource: + type: registry-image + source: + repository: concourse/oci-build-task + tag: 0.9.1 + registry_mirror: + host: ((registry-mirror-host)) + username: ((registry-mirror-username)) + password: ((registry-mirror-password)) +inputs: +- name: ci-images-git-repo +outputs: +- name: image +caches: +- path: ci-image-cache +params: + CONTEXT: ci-images-git-repo/ci/images + DOCKERFILE: ci-images-git-repo/ci/images/((ci-image-name))/Dockerfile + DOCKER_HUB_AUTH: ((docker-hub-auth)) +run: + path: /bin/sh + args: + - "-c" + - | + mkdir -p /root/.docker + cat > /root/.docker/config.json <