Moving pipelines to shepherd (#869)

2023-10-03 10:17:29 +02:00
parent 8df3e33838
commit 6f2453ac5a
6 changed files with 149 additions and 98 deletions
--- a/.editorconfig
+++ b/.editorconfig
@@ -1,15 +1,25 @@
 [*]
-charset=utf-8
-end_of_line=lf
-insert_final_newline=true
-indent_style=tab
-indent_size=4
+charset = utf-8
+end_of_line = lf
+insert_final_newline = true
+indent_style = tab
+indent_size = 4

-[*.{yml,yaml,sh}]
-indent_style=space
-indent_size=2
+[{spec,*.yml,*.yaml,Kilnfile,Kilnfile.lock}]
+indent_style = space
+indent_size = 2
+ij_yaml_keep_indents_on_empty_lines = false
+ij_yaml_keep_line_breaks = true
+
+[*.sh]
+indent_size = 2
+indent_style = space
+
+[{.asciidoctorconfig,*.ad,*.adoc,*.asciidoc}]
+ij_asciidoc_formatting_enabled = true
+ij_asciidoc_one_sentence_per_line = true

 [*.java]
 ij_java_class_count_to_use_import_on_demand = 100
-ij_java_imports_layout = jakarta.**,|,java.**,|,*,|,org.springframework.**,|,$*
+ij_java_imports_layout = jakarta.**, |, java.**, |, *, |, org.springframework.**, |, $*
 ij_java_names_count_to_use_import_on_demand = 100
--- a/ci/pipeline.yml
+++ b/ci/pipeline.yml
@@ -98,10 +98,12 @@ jobs:
          - get: git-repo
            passed: [build]
            trigger: true
-          - put: toolsmith-env
-            inputs: []
+          - put: cf-environment
            params:
-              action: claim
+              action: create
+              duration: 6h
+              resource: cf-environment
+            timeout: 6h
      - task: acceptance-tests
        file: git-repo/ci/tasks/acceptance-tests.yml
        vars:
@@ -109,11 +111,11 @@ jobs:
    on_success:
      in_parallel:
        - *slack-success-notification
-        - put: toolsmith-env
+        - put: cf-environment
          params:
-            action: unclaim
-            env_file: toolsmith-env/metadata
-          inputs: [toolsmith-env]
+            action: release
+            resource: cf-environment
+
    on_failure:
      *slack-failure-notification
    on_error:
@@ -244,11 +246,11 @@ resource_types:
      repository: ((dockerhub-mirror-registry))/cfcommunity/slack-notification-resource
      tag: latest

-  - name: pcf-pool
+  - name: shepherd
    type: registry-image
    source:
-      repository: ((dockerhub-mirror-registry))/cftoolsmiths/toolsmiths-envs-resource
-      tag: latest
+      repository: ((shepherd-image-repository))/shepherd-268822/shepherd2/concourse-resource
+      tag: v1

 resources:
  - name: nightly
@@ -323,13 +325,17 @@ resources:
    source:
      url: ((scs-slack-webhook))

-  - name: toolsmith-env
-    type: pcf-pool
-    icon: pool
+  - name: cf-environment
+    type: shepherd
    source:
-      api_token: ((toolsmith-api-token))
-      hostname: environments.toolsmiths.cf-app.com
-      pool_name: cf-deployment
+      url: https://v2.shepherd.run
+      service-account-key: ((shepherd-service-account-key))
+      lease:
+        namespace: spring-cloud-app-broker
+        pool:
+          namespace: official
+          name: cfd
+      compatibility-mode: environments-app

 groups:
  - name: "build"
--- a/ci/pr-pipeline.yml
+++ b/ci/pr-pipeline.yml
@@ -35,12 +35,14 @@ jobs:
    plan:
      - in_parallel:
          - get: pull-request
-            passed: [build]
+            passed: [ build ]
            trigger: true
-          - put: toolsmith-env
-            inputs: []
+          - put: cf-environment
            params:
-              action: claim
+              action: create
+              duration: 6h
+              resource: cf-environment
+            timeout: 6h
      - task: acceptance-tests
        file: pull-request/ci/tasks/acceptance-tests.yml
        vars:
@@ -50,11 +52,10 @@ jobs:
    on_success:
      in_parallel:
        - *slack-success-notification
-        - put: toolsmith-env
+        - put: cf-environment
          params:
-            action: unclaim
-            env_file: toolsmith-env/metadata
-          inputs: [ toolsmith-env ]
+            action: release
+            resource: cf-environment
    on_failure: *slack-failure-notification
    on_error: *slack-failure-notification

@@ -71,11 +72,11 @@ resource_types:
      repository: ((dockerhub-mirror-registry))/cfcommunity/slack-notification-resource
      tag: latest

-  - name: pcf-pool
+  - name: shepherd
    type: registry-image
    source:
-      repository: ((dockerhub-mirror-registry))/cftoolsmiths/toolsmiths-envs-resource
-      tag: latest
+      repository: ((shepherd-image-repository))/shepherd-268822/shepherd2/concourse-resource
+      tag: v1

 resources:
  - name: pull-request
@@ -95,10 +96,14 @@ resources:
    source:
      url: ((scs-slack-webhook))

-  - name: toolsmith-env
-    type: pcf-pool
-    icon: pool
+  - name: cf-environment
+    type: shepherd
    source:
-      api_token: ((toolsmith-api-token))
-      hostname: environments.toolsmiths.cf-app.com
-      pool_name: cf-deployment
+      url: https://v2.shepherd.run
+      service-account-key: ((shepherd-service-account-key))
+      lease:
+        namespace: spring-cloud-app-broker
+        pool:
+          namespace: official
+          name: cfd
+      compatibility-mode: environments-app
--- a/ci/scripts/acceptance-tests.sh
+++ b/ci/scripts/acceptance-tests.sh
@@ -2,37 +2,38 @@

 set -euo pipefail

-readonly TOOLSMITH_ENV_INPUT="${TOOLSMITH_ENV_INPUT:?must be set}"
-readonly DEFAULT_ORG="${DEFAULT_ORG:?must be set}"
-readonly DEFAULT_SPACE="${DEFAULT_SPACE:?must be set}"
-readonly SKIP_SSL_VALIDATION="${SKIP_SSL_VALIDATION:?must be set}"
-readonly ONLY_SHOW_STANDARD_STREAMS_ON_TEST_FAILURE="${ONLY_SHOW_STANDARD_STREAMS_ON_TEST_FAILURE:-true}"
-readonly DEPLOYMENT_DIRECTORY="$(mktemp -d)"
+readonly METADATA_FILE="${METADATA_FILE:?must be set}"
+
+declare CF_API_HOST
+readonly CF_USERNAME="admin"
+declare CF_PASSWORD

-declare API_HOST
-readonly API_PORT=443
-readonly USERNAME="admin"
-declare PASSWORD
 readonly CLIENT_ID="admin"
 declare CLIENT_SECRET

+readonly DEFAULT_ORG="test"
+readonly DEFAULT_SPACE="development"
+
+DEPLOYMENT_DIRECTORY="$(mktemp -d)"
+readonly DEPLOYMENT_DIRECTORY
+
 discover_environment() {
  local env_name
-  env_name=$(cat "$TOOLSMITH_ENV_INPUT/name")
+  env_name=$(jq -r .name <"${METADATA_FILE}")

-  eval "$(bbl print-env --metadata-file "$TOOLSMITH_ENV_INPUT/metadata")"
+  eval "$(bbl print-env --metadata-file "${METADATA_FILE}")"

-  API_HOST="$(jq -r .cf.api_url <"$TOOLSMITH_ENV_INPUT/metadata")"
-  PASSWORD="$(credhub get -n "/bosh-${env_name}/cf/cf_admin_password" -q)"
+  CF_API_HOST="$(jq -r .cf.api_url <"${METADATA_FILE}")"
+  CF_PASSWORD="$(credhub get -n "/bosh-${env_name}/cf/cf_admin_password" -q)"
  CLIENT_SECRET="$(credhub get -n "/bosh-${env_name}/cf/uaa_admin_client_secret" -q)"
 }

 prepare_cf_deployment() {
-  pushd "$DEPLOYMENT_DIRECTORY" > /dev/null
+  pushd "$DEPLOYMENT_DIRECTORY" >/dev/null

-  bosh --deployment cf manifest > manifest.yml
+  bosh --deployment cf manifest >manifest.yml

-  cat << EOF > ops.yml
+  cat <<EOF >ops.yml
 - type: replace
  path: /instance_groups/name=diego-cell/vm_type
  value: large
@@ -46,18 +47,13 @@ EOF

  bosh --non-interactive --deployment cf deploy --ops-file ops.yml manifest.yml

-  popd > /dev/null
+  popd >/dev/null
 }

 prepare_cf() {
  local -r test_instances_org="$DEFAULT_ORG-instances"

-  local skip_ssl_validation=""
-  if [ "$SKIP_SSL_VALIDATION" = "true" ]; then
-    skip_ssl_validation="--skip-ssl-validation"
-  fi
-
-  cf login -a "$API_HOST" -u "$USERNAME" -p "$PASSWORD" -o system "$skip_ssl_validation"
+  cf login -a "$CF_API_HOST" -u "$CF_USERNAME" -p "$CF_PASSWORD" -o system --skip-ssl-validation

  cf create-org "$DEFAULT_ORG"
  cf create-space "$DEFAULT_SPACE" -o "$DEFAULT_ORG"
@@ -67,27 +63,26 @@ prepare_cf() {
 }

 run_tests() {
-  export SPRING_CLOUD_APPBROKER_ACCEPTANCETEST_CLOUDFOUNDRY_API_HOST="${API_HOST}"
-  export SPRING_CLOUD_APPBROKER_ACCEPTANCETEST_CLOUDFOUNDRY_API_PORT="${API_PORT}"
-  export SPRING_CLOUD_APPBROKER_ACCEPTANCETEST_CLOUDFOUNDRY_USERNAME="${USERNAME}"
-  export SPRING_CLOUD_APPBROKER_ACCEPTANCETEST_CLOUDFOUNDRY_PASSWORD="${PASSWORD}"
+  export SPRING_CLOUD_APPBROKER_ACCEPTANCETEST_CLOUDFOUNDRY_API_HOST="${CF_API_HOST}"
+  export SPRING_CLOUD_APPBROKER_ACCEPTANCETEST_CLOUDFOUNDRY_API_PORT=443
+  export SPRING_CLOUD_APPBROKER_ACCEPTANCETEST_CLOUDFOUNDRY_USERNAME="${CF_USERNAME}"
+  export SPRING_CLOUD_APPBROKER_ACCEPTANCETEST_CLOUDFOUNDRY_PASSWORD="${CF_PASSWORD}"
  export SPRING_CLOUD_APPBROKER_ACCEPTANCETEST_CLOUDFOUNDRY_CLIENT_ID="${CLIENT_ID}"
  export SPRING_CLOUD_APPBROKER_ACCEPTANCETEST_CLOUDFOUNDRY_CLIENT_SECRET="${CLIENT_SECRET}"
  export SPRING_CLOUD_APPBROKER_ACCEPTANCETEST_CLOUDFOUNDRY_DEFAULT_ORG="${DEFAULT_ORG}"
  export SPRING_CLOUD_APPBROKER_ACCEPTANCETEST_CLOUDFOUNDRY_DEFAULT_SPACE="${DEFAULT_SPACE}"
-  export SPRING_CLOUD_APPBROKER_ACCEPTANCETEST_CLOUDFOUNDRY_SKIP_SSL_VALIDATION="${SKIP_SSL_VALIDATION}"
+  export SPRING_CLOUD_APPBROKER_ACCEPTANCETEST_CLOUDFOUNDRY_SKIP_SSL_VALIDATION=true
  export TESTS_BROKERAPPPATH=build/libs/spring-cloud-app-broker-acceptance-tests.jar
  ./gradlew -PacceptanceTests \
-    -PonlyShowStandardStreamsOnTestFailure="${ONLY_SHOW_STANDARD_STREAMS_ON_TEST_FAILURE}" \
+    -PonlyShowStandardStreamsOnTestFailure=true \
    :spring-cloud-app-broker-acceptance-tests:test
 }

 main() {
  discover_environment

-  echo "Running tests against $API_HOST"
+  echo "Running tests against ${CF_API_HOST}"
  echo
-
  prepare_cf_deployment
  prepare_cf

--- a/ci/tasks/acceptance-tests.yml
+++ b/ci/tasks/acceptance-tests.yml
@@ -9,13 +9,9 @@ image_resource:
    tag: ((ci-image-tag))
 inputs:
  - name: git-repo
-  - name: toolsmith-env
+  - name: cf-environment
 run:
  path: ci/scripts/acceptance-tests.sh
  dir: git-repo
 params:
-  TOOLSMITH_ENV_INPUT: ../toolsmith-env
-  DEFAULT_ORG: ((acceptance-test-default-org))
-  DEFAULT_SPACE: ((acceptance-test-default-space))
-  SKIP_SSL_VALIDATION: ((acceptance-test-skip-ssl-validation))
-  ONLY_SHOW_STANDARD_STREAMS_ON_TEST_FAILURE: true
+  METADATA_FILE: ../cf-environment/metadata
--- a/scripts/acceptance-tests-ide.sh
+++ b/scripts/acceptance-tests-ide.sh
@@ -2,46 +2,85 @@

 set -euo pipefail

-readonly TOOLSMITH_ENV_INPUT="${TOOLSMITH_ENV_INPUT:?must be set}"
-readonly DEFAULT_ORG="${DEFAULT_ORG:?must be set}"
-readonly DEFAULT_SPACE="${DEFAULT_SPACE:?must be set}"
-readonly SKIP_SSL_VALIDATION="${SKIP_SSL_VALIDATION:?must be set}"
+readonly METADATA_FILE="${METADATA_FILE:?must be set}"
+
+declare CF_API_HOST
+readonly CF_USERNAME="admin"
+declare CF_PASSWORD

-declare API_HOST
-readonly API_PORT=443
-readonly USERNAME="admin"
-declare PASSWORD
 readonly CLIENT_ID="admin"
 declare CLIENT_SECRET

+readonly DEFAULT_ORG="test"
+readonly DEFAULT_SPACE="development"
+
+DEPLOYMENT_DIRECTORY="$(mktemp -d)"
+readonly DEPLOYMENT_DIRECTORY
+
 discover_environment() {
  local env_name
-  env_name=$(cat "$TOOLSMITH_ENV_INPUT/name")
+  env_name=$(jq -r .name <"${METADATA_FILE}")

-  eval "$(bbl print-env --metadata-file "$TOOLSMITH_ENV_INPUT/metadata")"
+  eval "$(bbl print-env --metadata-file "${METADATA_FILE}")"

-  API_HOST="$(jq -r .cf.api_url <"$TOOLSMITH_ENV_INPUT/metadata")"
-  PASSWORD="$(credhub get -n "/bosh-${env_name}/cf/cf_admin_password" -q)"
+  CF_API_HOST="$(jq -r .cf.api_url <"${METADATA_FILE}")"
+  CF_PASSWORD="$(credhub get -n "/bosh-${env_name}/cf/cf_admin_password" -q)"
  CLIENT_SECRET="$(credhub get -n "/bosh-${env_name}/cf/uaa_admin_client_secret" -q)"
 }

+prepare_cf_deployment() {
+  pushd "$DEPLOYMENT_DIRECTORY" >/dev/null
+
+  bosh --deployment cf manifest >manifest.yml
+
+  cat <<EOF >ops.yml
+- type: replace
+  path: /instance_groups/name=diego-cell/vm_type
+  value: large
+- type: replace
+  path: /instance_groups/name=router/vm_type
+  value: large
+- type: replace
+  path: /instance_groups/name=api/vm_type
+  value: large
+EOF
+
+  bosh --non-interactive --deployment cf deploy --ops-file ops.yml manifest.yml
+
+  popd >/dev/null
+}
+
+prepare_cf() {
+  local -r test_instances_org="$DEFAULT_ORG-instances"
+
+  cf login -a "$CF_API_HOST" -u "$CF_USERNAME" -p "$CF_PASSWORD" -o system --skip-ssl-validation
+
+  cf create-org "$DEFAULT_ORG"
+  cf create-space "$DEFAULT_SPACE" -o "$DEFAULT_ORG"
+
+  cf create-org "$test_instances_org"
+  cf create-space "$DEFAULT_SPACE" -o "$test_instances_org"
+}
+
 print_values() {
-cat <<EOF
-SPRING_CLOUD_APPBROKER_ACCEPTANCETEST_CLOUDFOUNDRY_API_HOST=${API_HOST}
-SPRING_CLOUD_APPBROKER_ACCEPTANCETEST_CLOUDFOUNDRY_API_PORT=${API_PORT}
-SPRING_CLOUD_APPBROKER_ACCEPTANCETEST_CLOUDFOUNDRY_USERNAME=${USERNAME}
-SPRING_CLOUD_APPBROKER_ACCEPTANCETEST_CLOUDFOUNDRY_PASSWORD=${PASSWORD}
+  cat <<EOF
+SPRING_CLOUD_APPBROKER_ACCEPTANCETEST_CLOUDFOUNDRY_API_HOST=${CF_API_HOST}
+SPRING_CLOUD_APPBROKER_ACCEPTANCETEST_CLOUDFOUNDRY_API_PORT=443
+SPRING_CLOUD_APPBROKER_ACCEPTANCETEST_CLOUDFOUNDRY_USERNAME=${CF_USERNAME}
+SPRING_CLOUD_APPBROKER_ACCEPTANCETEST_CLOUDFOUNDRY_PASSWORD=${CF_PASSWORD}
 SPRING_CLOUD_APPBROKER_ACCEPTANCETEST_CLOUDFOUNDRY_CLIENT_ID=${CLIENT_ID}
 SPRING_CLOUD_APPBROKER_ACCEPTANCETEST_CLOUDFOUNDRY_CLIENT_SECRET=${CLIENT_SECRET}
 SPRING_CLOUD_APPBROKER_ACCEPTANCETEST_CLOUDFOUNDRY_DEFAULT_ORG=${DEFAULT_ORG}
 SPRING_CLOUD_APPBROKER_ACCEPTANCETEST_CLOUDFOUNDRY_DEFAULT_SPACE=${DEFAULT_SPACE}
-SPRING_CLOUD_APPBROKER_ACCEPTANCETEST_CLOUDFOUNDRY_SKIP_SSL_VALIDATION=${SKIP_SSL_VALIDATION}
+SPRING_CLOUD_APPBROKER_ACCEPTANCETEST_CLOUDFOUNDRY_SKIP_SSL_VALIDATION=true
 TESTS_BROKERAPPPATH=build/libs/spring-cloud-app-broker-acceptance-tests.jar
 EOF
 }

 main() {
  discover_environment
+  prepare_cf_deployment
+  prepare_cf
  print_values
 }