From 15de06d33a06487ea7f91996abad3b088a6f9eb4 Mon Sep 17 00:00:00 2001 From: Brian Clozel Date: Fri, 13 Nov 2020 11:51:14 +0100 Subject: [PATCH] Add Concourse CI pipeline This commit adds the Concourse CI pipeline configuration. The CI build is hosted on https://ci.spring.io. As an experimental project, artifacts are published for now using the org.springframework.experimental groupId. Closes gh-18 --- ci/README.adoc | 51 +++++++++++ ci/images/README.adoc | 21 +++++ ci/images/get-jdk-url.sh | 17 ++++ ci/images/setup.sh | 32 +++++++ ci/images/spring-graphql-ci-image/Dockerfile | 8 ++ ci/parameters.yml | 9 ++ ci/pipeline.yml | 91 ++++++++++++++++++++ ci/scripts/build-project.sh | 9 ++ ci/scripts/common.sh | 2 + ci/tasks/build-project.yml | 22 +++++ 10 files changed, 262 insertions(+) create mode 100644 ci/README.adoc create mode 100644 ci/images/README.adoc create mode 100755 ci/images/get-jdk-url.sh create mode 100755 ci/images/setup.sh create mode 100644 ci/images/spring-graphql-ci-image/Dockerfile create mode 100644 ci/parameters.yml create mode 100644 ci/pipeline.yml create mode 100755 ci/scripts/build-project.sh create mode 100644 ci/scripts/common.sh create mode 100644 ci/tasks/build-project.yml diff --git a/ci/README.adoc b/ci/README.adoc new file mode 100644 index 00000000..004eaf37 --- /dev/null +++ b/ci/README.adoc @@ -0,0 +1,51 @@ +== Spring GraphQL Concourse pipeline + +The Spring GraphQL experimental project is using https://concourse-ci.org/[Concourse] for its CI build and other automated tasks. +The Spring team has a dedicated Concourse instance available at https://ci.spring.io. + +=== Setting up your development environment + +If you're part of the Spring GraphQL team on GitHub, you can get access to CI management features. +First, you need to go to https://ci.spring.io and install the client CLI for your platform (see bottom right of the screen). + +You can then login with the instance using: + +[source] +---- +$ fly -t spring-graphql login -n spring-graphql -c https://ci.spring.io +---- + +Once logged in, you should get something like: + +[source] +---- +$ fly ts +name url team expiry +spring-graphql https://ci.spring.io spring-graphql Wed, 25 Mar 2020 17:45:26 UTC +---- + +=== Pipeline configuration and structure +The build pipelines are described in `pipeline.yml` file. +This file is listing Concourse resources, i.e. build inputs and outputs such as container images, artifact repositories, source repositories, notification services, etc. +It also describes jobs (a job is a sequence of inputs, tasks and outputs); jobs are organized by groups. + +The `pipeline.yml` definition contains `((parameters))` which are loaded from the `parameters.yml` file or from our https://docs.cloudfoundry.org/credhub/[credhub instance]. + +You'll find in this folder the following resources: +* `pipeline.yml` the build pipeline +* `parameters.yml` the build parameters used for the pipeline +* `images/` holds the container images definitions used in this pipeline +* `scripts/` holds the build scripts that ship within the CI container images +* `tasks` contains the task definitions used in the main `pipeline.yml` + +=== Updating the build pipeline + +Updating files on the repository is not enough to update the build pipeline, as changes need to be applied. +The pipeline can be deployed using the following command: + +[source] +---- +$ fly -t spring-graphql set-pipeline -p spring-graphql -c ci/pipeline.yml -l ci/parameters.yml +---- + +NOTE: This assumes that you have credhub integration configured with the appropriate secrets. \ No newline at end of file diff --git a/ci/images/README.adoc b/ci/images/README.adoc new file mode 100644 index 00000000..07fb98b4 --- /dev/null +++ b/ci/images/README.adoc @@ -0,0 +1,21 @@ +== CI Images + +These images are used by CI to run the actual builds. + +To build the image locally run the following from this directory: + +---- +$ docker build --no-cache -f /Dockerfile . +---- + +For example + +---- +$ docker build --no-cache -f spring-graphql-ci-image/Dockerfile . +---- + +To test run: + +---- +$ docker run -it --entrypoint /bin/bash +---- diff --git a/ci/images/get-jdk-url.sh b/ci/images/get-jdk-url.sh new file mode 100755 index 00000000..eec7c536 --- /dev/null +++ b/ci/images/get-jdk-url.sh @@ -0,0 +1,17 @@ +#!/bin/bash +set -e + +case "$1" in + java8) + echo "https://github.com/AdoptOpenJDK/openjdk8-binaries/releases/download/jdk8u272-b10/OpenJDK8U-jdk_x64_linux_hotspot_8u272b10.tar.gz" + ;; + java11) + echo "https://github.com/AdoptOpenJDK/openjdk11-binaries/releases/download/jdk-11.0.9%2B11.1/OpenJDK11U-jdk_x64_linux_hotspot_11.0.9_11.tar.gz" + ;; + java15) + echo "https://github.com/AdoptOpenJDK/openjdk15-binaries/releases/download/jdk-15.0.1%2B9/OpenJDK15U-jdk_x64_linux_hotspot_15.0.1_9.tar.gz" + ;; + *) + echo $"Unknown java version" + exit 1 +esac \ No newline at end of file diff --git a/ci/images/setup.sh b/ci/images/setup.sh new file mode 100755 index 00000000..ffc3d86d --- /dev/null +++ b/ci/images/setup.sh @@ -0,0 +1,32 @@ +#!/bin/bash +set -ex + +########################################################### +# UTILS +########################################################### + +apt-get update +apt-get install --no-install-recommends -y ca-certificates net-tools libxml2-utils git curl libudev1 libxml2-utils iptables iproute2 jq fontconfig +rm -rf /var/lib/apt/lists/* + +curl https://raw.githubusercontent.com/spring-io/concourse-java-scripts/v0.0.3/concourse-java.sh > /opt/concourse-java.sh + +curl --output /opt/concourse-release-scripts.jar https://repo.spring.io/release/io/spring/concourse/releasescripts/concourse-release-scripts/0.2.1/concourse-release-scripts-0.2.1.jar + +########################################################### +# JAVA +########################################################### +JDK_URL=$( ./get-jdk-url.sh $1 ) + +mkdir -p /opt/openjdk +cd /opt/openjdk +curl -L ${JDK_URL} | tar zx --strip-components=1 +test -f /opt/openjdk/bin/java +test -f /opt/openjdk/bin/javac + +########################################################### +# GRADLE ENTERPRISE +########################################################### +cd / +mkdir ~/.gradle +echo 'systemProp.user.name=concourse' > ~/.gradle/gradle.properties diff --git a/ci/images/spring-graphql-ci-image/Dockerfile b/ci/images/spring-graphql-ci-image/Dockerfile new file mode 100644 index 00000000..4d04ecca --- /dev/null +++ b/ci/images/spring-graphql-ci-image/Dockerfile @@ -0,0 +1,8 @@ +FROM ubuntu:focal-20201008 + +ADD setup.sh /setup.sh +ADD get-jdk-url.sh /get-jdk-url.sh +RUN ./setup.sh java8 + +ENV JAVA_HOME /opt/openjdk +ENV PATH $JAVA_HOME/bin:$PATH diff --git a/ci/parameters.yml b/ci/parameters.yml new file mode 100644 index 00000000..02ea14da --- /dev/null +++ b/ci/parameters.yml @@ -0,0 +1,9 @@ +github-repo: "https://github.com/spring-projects-experimental/spring-graphql.git" +github-repo-name: "spring-projects-experimental/spring-graphql" +docker-hub-organization: "springci" +artifactory-server: "https://repo.spring.io" +branch: "master" +build-name: "spring-graphql" +pipeline-name: "spring-graphql" +concourse-url: "https://ci.spring.io" +task-timeout: 45m \ No newline at end of file diff --git a/ci/pipeline.yml b/ci/pipeline.yml new file mode 100644 index 00000000..5b20a907 --- /dev/null +++ b/ci/pipeline.yml @@ -0,0 +1,91 @@ +anchors: + docker-resource-source: &docker-resource-source + username: ((docker-hub-username)) + password: ((docker-hub-password)) + tag: master + gradle-enterprise-task-params: &gradle-enterprise-task-params + GRADLE_ENTERPRISE_ACCESS_KEY: ((gradle_enterprise_secret_access_key)) + GRADLE_ENTERPRISE_CACHE_USERNAME: ((gradle_enterprise_cache_user.username)) + GRADLE_ENTERPRISE_CACHE_PASSWORD: ((gradle_enterprise_cache_user.password)) + +resource_types: +- name: artifactory-resource + type: docker-image + source: + repository: springio/artifactory-resource + tag: 0.0.12 + +resources: +- name: git-repo + type: git + icon: github + source: + uri: ((github-repo)) + username: ((github-username)) + password: ((github-password)) + branch: ((branch)) + ignore_paths: ["ci/*"] +- name: ci-images-git-repo + type: git + icon: github + source: + uri: ((github-repo)) + branch: ((branch)) + paths: ["ci/images/*"] +- name: spring-graphql-ci-image + type: docker-image + icon: docker + source: + <<: *docker-resource-source + repository: ((docker-hub-organization))/spring-graphql-ci-image +- name: artifactory-repo + type: artifactory-resource + icon: package-variant + source: + uri: ((artifactory-server)) + username: ((artifactory-username)) + password: ((artifactory-password)) + build_name: ((build-name)) + +jobs: +- name: build-spring-graphql-ci-images + plan: + - get: ci-images-git-repo + trigger: true + - in_parallel: + - put: spring-graphql-ci-image + params: + build: ci-images-git-repo/ci/images + dockerfile: ci-images-git-repo/ci/images/spring-graphql-ci-image/Dockerfile +- name: build + serial: true + public: true + plan: + - get: spring-graphql-ci-image + - get: git-repo + trigger: true + - do: + - task: build-project + privileged: true + timeout: ((task-timeout)) + image: spring-graphql-ci-image + file: git-repo/ci/tasks/build-project.yml + params: + BRANCH: ((branch)) + <<: *gradle-enterprise-task-params + - put: artifactory-repo + params: &artifactory-params + repo: libs-snapshot-local + folder: distribution-repository + build_uri: "https://ci.spring.io/teams/${BUILD_TEAM_NAME}/pipelines/${BUILD_PIPELINE_NAME}/jobs/${BUILD_JOB_NAME}/builds/${BUILD_NAME}" + build_number: "${BUILD_PIPELINE_NAME}-${BUILD_JOB_NAME}-${BUILD_NAME}" + disable_checksum_uploads: true + threads: 8 + get_params: + threads: 8 + +groups: +- name: "builds" + jobs: ["build"] +- name: "ci-images" + jobs: ["build-spring-graphql-ci-images"] diff --git a/ci/scripts/build-project.sh b/ci/scripts/build-project.sh new file mode 100755 index 00000000..3844d1a3 --- /dev/null +++ b/ci/scripts/build-project.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -e + +source $(dirname $0)/common.sh +repository=$(pwd)/distribution-repository + +pushd git-repo > /dev/null +./gradlew -Dorg.gradle.internal.launcher.welcomeMessageEnabled=false --no-daemon --max-workers=4 -PdeploymentRepository=${repository} build publishAllPublicationsToDeploymentRepository +popd > /dev/null diff --git a/ci/scripts/common.sh b/ci/scripts/common.sh new file mode 100644 index 00000000..1accaa61 --- /dev/null +++ b/ci/scripts/common.sh @@ -0,0 +1,2 @@ +source /opt/concourse-java.sh +setup_symlinks \ No newline at end of file diff --git a/ci/tasks/build-project.yml b/ci/tasks/build-project.yml new file mode 100644 index 00000000..759749ef --- /dev/null +++ b/ci/tasks/build-project.yml @@ -0,0 +1,22 @@ +--- +platform: linux +inputs: +- name: git-repo +outputs: +- name: distribution-repository +- name: git-repo +caches: +- path: gradle +params: + BRANCH: + CI: true + GRADLE_ENTERPRISE_ACCESS_KEY: + GRADLE_ENTERPRISE_CACHE_USERNAME: + GRADLE_ENTERPRISE_CACHE_PASSWORD: + GRADLE_ENTERPRISE_URL: https://ge.spring.io +run: + path: bash + args: + - -ec + - | + ${PWD}/git-repo/ci/scripts/build-project.sh