From d609cc4c115d22f0641ec9c7e8d43dd340fdb700 Mon Sep 17 00:00:00 2001 From: Gareth Clay Date: Tue, 29 Nov 2022 16:12:53 +0000 Subject: [PATCH] Instanced pipelines Use instanced pipelines for supported branches and PRs. --- .editorconfig | 2 +- ci/images/app-broker-ci/Dockerfile | 23 ++++++-- ci/pipeline.yml | 85 +++++++++++++++++++----------- ci/pr-manager-pipeline.yml | 64 ++++++++++++++++++++++ ci/pr-pipeline.yml | 45 ++++++++-------- ci/tasks/list-prs.yml | 27 ++++++++++ scripts/set-pipelines.sh | 83 ++++++++++++++++++++++------- 7 files changed, 251 insertions(+), 78 deletions(-) create mode 100644 ci/pr-manager-pipeline.yml create mode 100644 ci/tasks/list-prs.yml diff --git a/.editorconfig b/.editorconfig index ede4e0b..11a63aa 100644 --- a/.editorconfig +++ b/.editorconfig @@ -5,6 +5,6 @@ insert_final_newline=true indent_style=tab indent_size=4 -[*.{yml,yaml}] +[*.{yml,yaml,sh}] indent_style=space indent_size=2 diff --git a/ci/images/app-broker-ci/Dockerfile b/ci/images/app-broker-ci/Dockerfile index a349d2f..8604cb4 100644 --- a/ci/images/app-broker-ci/Dockerfile +++ b/ci/images/app-broker-ci/Dockerfile @@ -1,12 +1,25 @@ FROM harbor-repo.vmware.com/dockerhub-proxy-cache/library/ubuntu:bionic -RUN apt-get update && apt-get install --no-install-recommends -y ca-certificates net-tools git curl jq gnupg +RUN apt-get update && apt-get install --no-install-recommends -y \ + ca-certificates \ + curl \ + git \ + gnupg \ + jq \ + net-tools -RUN curl -L https://packages.cloudfoundry.org/debian/cli.cloudfoundry.org.key | apt-key add - -RUN echo "deb https://packages.cloudfoundry.org/debian stable main" | tee /etc/apt/sources.list.d/cloudfoundry-cli.list -RUN apt-get update && apt-get install cf-cli +RUN mkdir -p /etc/apt/keyrings +RUN curl -L 'https://cli.github.com/packages/githubcli-archive-keyring.gpg' | gpg --dearmor -o /etc/apt/keyrings/githubcli-archive-keyring.gpg +RUN curl -L 'https://packages.cloudfoundry.org/debian/cli.cloudfoundry.org.key' | gpg --dearmor -o /etc/apt/keyrings/cloudfoundry-cli-keyring.gpg -RUN rm -rf /var/lib/apt/lists/* +RUN echo "deb [signed-by=/etc/apt/keyrings/cloudfoundry-cli-keyring.gpg] https://packages.cloudfoundry.org/debian stable main" | tee /etc/apt/sources.list.d/cloudfoundry-cli.list +RUN echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages stable main" | tee /etc/apt/sources.list.d/github-cli.list + +RUN apt-get update && \ + apt-get -qy --no-install-recommends install \ + cf-cli \ + gh && \ + apt-get clean ENV JAVA_HOME /opt/openjdk ENV PATH $JAVA_HOME/bin:$PATH diff --git a/ci/pipeline.yml b/ci/pipeline.yml index 364f6c9..ca4cbf5 100644 --- a/ci/pipeline.yml +++ b/ci/pipeline.yml @@ -6,25 +6,47 @@ aliases: icon_emoji: ":concourse:" username: concourse channel: ((ab-slack-success-channel)) - text: "$BUILD_PIPELINE_NAME pipeline has succeeded with build <${ATC_EXTERNAL_URL}/builds/$BUILD_ID|$BUILD_NAME>!" + text: "`$BUILD_PIPELINE_NAME ((branch))` pipeline has succeeded with build <${ATC_EXTERNAL_URL}/builds/$BUILD_ID|$BUILD_NAME>!" - &slack-failure-notification put: alert params: icon_emoji: ":animal-1252:" username: concourse channel: ((ab-slack-failure-channel)) - text: Build <${ATC_EXTERNAL_URL}/builds/$BUILD_ID|$BUILD_NAME> of job $BUILD_JOB_NAME in the $BUILD_PIPELINE_NAME pipeline has failed! + text: " Build <${ATC_EXTERNAL_URL}/builds/$BUILD_ID|$BUILD_NAME> of job `$BUILD_JOB_NAME` in the `$BUILD_PIPELINE_NAME ((branch))` pipeline has failed!" jobs: - name: build-ci-images plan: - get: ci-images-git-repo trigger: true - - put: app-broker-ci-image + - task: build-image + privileged: true + tags: [ btrfs ] + config: + platform: linux + image_resource: + type: registry-image + source: + repository: ((dockerhub-mirror-registry))/concourse/oci-build-task + tag: 0.10.0 + inputs: + - name: source + outputs: + - name: image + run: + path: build + input_mapping: + source: ci-images-git-repo params: - build: ci-images-git-repo/ci/images/app-broker-ci + CONTEXT: source/ci/images/app-broker-ci + - put: app-broker-ci-image + inputs: detect + params: + image: image/image.tar get_params: - skip_download: "true" + skip_download: true + on_failure: *slack-failure-notification - name: build serial: true @@ -57,16 +79,16 @@ jobs: serial: true plan: - in_parallel: - - get: nightly - passed: [build] - trigger: true - - get: git-repo - passed: [build] - trigger: true - - put: toolsmith-env - inputs: [] - params: - action: claim + - get: nightly + passed: [build] + trigger: true + - get: git-repo + passed: [build] + trigger: true + - put: toolsmith-env + inputs: [] + params: + action: claim - task: acceptance-tests file: git-repo/ci/tasks/acceptance-tests.yml vars: @@ -156,11 +178,11 @@ jobs: serial: true plan: - in_parallel: - - get: git-repo - - get: artifactory-repo - passed: [stage-rc] - params: - save_build_info: true + - get: git-repo + - get: artifactory-repo + passed: [stage-rc] + params: + save_build_info: true - task: promote file: git-repo/ci/tasks/promote.yml vars: @@ -171,11 +193,11 @@ jobs: serial: true plan: - in_parallel: - - get: git-repo - - get: artifactory-repo - passed: [stage-release] - params: - save_build_info: true + - get: git-repo + - get: artifactory-repo + passed: [stage-release] + params: + save_build_info: true - task: promote file: git-repo/ci/tasks/promote.yml vars: @@ -186,11 +208,11 @@ jobs: serial: true plan: - in_parallel: - - get: git-repo - - get: artifactory-repo - passed: [promote-release] - params: - save_build_info: true + - get: git-repo + - get: artifactory-repo + passed: [promote-release] + params: + save_build_info: true - task: sync-to-maven-central file: git-repo/ci/tasks/sync-to-maven-central.yml vars: @@ -229,6 +251,7 @@ resources: - Thursday - Friday location: America/New_York + initial_version: true - name: git-repo type: git @@ -256,7 +279,7 @@ resources: paths: ["ci/images/*"] - name: app-broker-ci-image - type: docker-image + type: registry-image source: repository: ((corporate-harbor-registry))/((dockerhub-organization))/app-broker-ci username: ((corporate-harbor-robot-account.username)) diff --git a/ci/pr-manager-pipeline.yml b/ci/pr-manager-pipeline.yml new file mode 100644 index 0000000..b9f3504 --- /dev/null +++ b/ci/pr-manager-pipeline.yml @@ -0,0 +1,64 @@ +jobs: + - name: manage-pr-pipelines + plan: + - in_parallel: + - get: pipeline-definition + trigger: true + - get: pr + trigger: true + version: every + - task: list-prs + file: pipeline-definition/ci/tasks/list-prs.yml + input_mapping: + git-repo: pipeline-definition + vars: + ci-image-tag: ((ci-image-tag)) + - load_var: prs + file: prs/prs.json + - across: + - var: pr + values: ((.:prs)) + max_in_flight: all + in_parallel: + - set_pipeline: ((pipeline-name)) + file: pipeline-definition/ci/pr-pipeline.yml + var_files: + - pipeline-definition/ci/config-concourse.yml + vars: + ci-image-tag: ((.:pr.baseRefName)) + pr: + number: ((.:pr.number)) + commit: ((.:pr.headRefOid)) + base-branch: ((.:pr.baseRefName)) + instance_vars: + pr: ((.:pr.number)) + branch: ((.:pr.baseRefName)) + author: ((.:pr.author.login)) + +resource_types: + - name: pull-request + type: registry-image + source: + repository: ((dockerhub-mirror-registry))/teliaoss/github-pr-resource + tag: v0.23.0 + +resources: + - name: pipeline-definition + type: git + icon: github + source: + uri: ((github-repo)) + username: ((github-token)) + password: ((github-password)) + branch: main + paths: + - ci/pr-pipeline.yml + - ci/config-concourse.yml + + - name: pr + type: pull-request + icon: github + source: + repository: ((app-broker-pr-github-pull-requests-repo)) + access_token: ((app-broker-pr-github-pull-requests-access-token)) + required_review_approvals: 1 diff --git a/ci/pr-pipeline.yml b/ci/pr-pipeline.yml index 4d25fe7..bc3c603 100644 --- a/ci/pr-pipeline.yml +++ b/ci/pr-pipeline.yml @@ -6,14 +6,15 @@ aliases: icon_emoji: ":concourse:" username: concourse channel: ((ab-slack-success-channel)) - text: "$BUILD_PIPELINE_NAME pipeline has succeeded with build <${ATC_EXTERNAL_URL}/builds/$BUILD_ID|$BUILD_NAME>!" + text: "`$BUILD_PIPELINE_NAME` pipeline has succeeded with build <${ATC_EXTERNAL_URL}/builds/$BUILD_ID|$BUILD_NAME>!" + - &slack-failure-notification put: alert params: icon_emoji: ":animal-1252:" username: concourse channel: ((ab-slack-failure-channel)) - text: Build <${ATC_EXTERNAL_URL}/builds/$BUILD_ID|$BUILD_NAME> of job $BUILD_JOB_NAME in the $BUILD_PIPELINE_NAME pipeline has failed! + text: " Build <${ATC_EXTERNAL_URL}/builds/$BUILD_ID|$BUILD_NAME> of job `$BUILD_JOB_NAME` in the `$BUILD_PIPELINE_NAME` pipeline has failed!" jobs: - name: build @@ -28,17 +29,18 @@ jobs: git-repo: pull-request vars: ci-image-tag: ((ci-image-tag)) + - name: run-acceptance-tests serial: true plan: - in_parallel: - - get: pull-request - passed: [build] - trigger: true - - put: toolsmith-env - inputs: [ ] - params: - action: claim + - get: pull-request + passed: [build] + trigger: true + - put: toolsmith-env + inputs: [] + params: + action: claim - task: acceptance-tests file: pull-request/ci/tasks/acceptance-tests.yml vars: @@ -47,16 +49,14 @@ jobs: git-repo: pull-request on_success: in_parallel: - - *slack-success-notification - - put: toolsmith-env - params: - action: unclaim - env_file: toolsmith-env/metadata - inputs: [ toolsmith-env ] - on_failure: - *slack-failure-notification - on_error: - *slack-failure-notification + - *slack-success-notification + - put: toolsmith-env + params: + action: unclaim + env_file: toolsmith-env/metadata + inputs: [ toolsmith-env ] + on_failure: *slack-failure-notification + on_error: *slack-failure-notification resource_types: - name: pull-request @@ -80,11 +80,14 @@ resource_types: resources: - name: pull-request type: pull-request - check_every: 10s + version: + pr: ((pr.number)) + commit: ((pr.commit)) source: repository: ((app-broker-pr-github-pull-requests-repo)) access_token: ((app-broker-pr-github-pull-requests-access-token)) - base_branch: ((branch)) + base_branch: ((pr.base-branch)) + required_review_approvals: 1 - name: alert type: slack-notification diff --git a/ci/tasks/list-prs.yml b/ci/tasks/list-prs.yml new file mode 100644 index 0000000..1009520 --- /dev/null +++ b/ci/tasks/list-prs.yml @@ -0,0 +1,27 @@ +platform: linux + +image_resource: + type: registry-image + source: + repository: ((corporate-harbor-registry))/((dockerhub-organization))/app-broker-ci + tag: ((ci-image-tag)) + username: ((corporate-harbor-robot-account.username)) + password: ((corporate-harbor-robot-account.password)) + +inputs: + - name: git-repo + +outputs: + - name: prs + +run: + path: bash + dir: git-repo + args: + - -c + - | + gh pr list --state open --search "review:approved" + gh pr list --state open --search "review:approved" --json number,baseRefName,headRefOid,author > ../prs/prs.json + +params: + GH_TOKEN: ((app-broker-pr-github-pull-requests-access-token)) diff --git a/scripts/set-pipelines.sh b/scripts/set-pipelines.sh index 7a6edac..b6a4ce3 100755 --- a/scripts/set-pipelines.sh +++ b/scripts/set-pipelines.sh @@ -3,35 +3,78 @@ set -euo pipefail readonly FLY_TARGET="app-broker" -readonly VERSION=1.6.x -readonly BRANCH=main -readonly CI_IMAGE_TAG=main +readonly PIPELINE_NAME_SUFFIX="${PIPELINE_NAME_SUFFIX:-""}" +readonly PIPELINE_TYPE=${1:-""} -set_pipeline() { - local pipeline_name pipeline_definition branch ci_image_tag - pipeline_name="${1:?pipeline name must be provided}" - pipeline_definition="${2:?pipeline definition file must be provided}" - branch="${3:?branch must be provided}" - ci_image_tag="${4:?ci_image_tag must be provided}" +[[ "$PIPELINE_NAME_SUFFIX" ]] && DISABLE_SLACK_ALERTING="true" || DISABLE_SLACK_ALERTING="false" - echo "Setting $pipeline_name pipeline..." - fly --target "$FLY_TARGET" set-pipeline \ - --pipeline "$pipeline_name" \ - --config "$pipeline_definition" \ +set_branch_pipeline() { + local -r pipeline_name="app-broker${PIPELINE_NAME_SUFFIX:+"-$PIPELINE_NAME_SUFFIX"}" + local -r branches=("main" "1.6.x" "1.5.x" "1.4.x") + + for branch in "${branches[@]}"; do + echo "Setting $pipeline_name $branch pipeline..." + + fly --target "$FLY_TARGET" set-pipeline \ + --pipeline "$pipeline_name" \ + --config pipeline.yml \ + --load-vars-from config-concourse.yml \ + --instance-var "branch=$branch" \ + --var "ci-image-tag=$branch" \ + --var "disable-slack-alerting=$DISABLE_SLACK_ALERTING" \` + done +} + +set_pr_manager_pipeline() { + local -r pipeline_name="app-broker-pull-requests${PIPELINE_NAME_SUFFIX:+"-$PIPELINE_NAME_SUFFIX"}" + + echo "Setting PR manager pipeline..." + + fly --target "$FLY_TARGET" set-pipeline --pipeline "$pipeline_name" \ + --config pr-manager-pipeline.yml \ --load-vars-from config-concourse.yml \ - --var "branch=$branch" \ - --var "ci-image-tag=$ci_image_tag" + --var ci-image-tag="main" \ + --var pipeline-name="$pipeline_name" +} + +set_pr_pipeline() { + local -r pr_number="${1:?first argument must be the PR number}" + local -r pipeline_name="app-broker-pr-$pr_number${PIPELINE_NAME_SUFFIX:+"-$PIPELINE_NAME_SUFFIX"}" + + echo "Setting PR pipeline for PR $pr_number..." + + local -r pr_info="$(gh pr view "$pr_number" --json baseRefName,headRefOid)" + + fly --target "$FLY_TARGET" set-pipeline --pipeline "$pipeline_name" \ + --config pr-pipeline.yml \ + --load-vars-from config-concourse.yml \ + --var ci-image-tag="pr-test" \ + --var pipeline-name="$pipeline_name" \ + --var pr.number="$pr_number" \ + --var pr.base-branch="$(jq -r '.baseRefName' <<< "$pr_info")" \ + --var pr.commit="$(jq -r '.headRefOid' <<< "$pr_info")" } main() { - fly -t app-broker sync - pushd "$(dirname "$0")/../ci" >/dev/null - set_pipeline "app-broker-$VERSION" pipeline.yml "$BRANCH" "$CI_IMAGE_TAG" - set_pipeline "app-broker-$VERSION-pr" pr-pipeline.yml "$BRANCH" "$CI_IMAGE_TAG" + case "${PIPELINE_TYPE#--}" in + "branch") + set_branch_pipeline + ;; + "pr-manager") + set_pr_manager_pipeline + ;; + "pr") + set_pr_pipeline "$2" + ;; + *) + set_branch_pipeline + set_pr_manager_pipeline + ;; + esac popd >/dev/null } -main +main "$@"