diff --git a/.github/workflows/common.yml b/.github/workflows/common.yml index 97ff05a4..e28bffc5 100644 --- a/.github/workflows/common.yml +++ b/.github/workflows/common.yml @@ -13,6 +13,11 @@ on: required: false default: true description: 'Verbose output' + scale: + type: boolean + required: false + default: false + description: 'Scale and use private runners' secrets: DOCKER_HUB_USERNAME: DOCKER_HUB_PASSWORD: @@ -43,13 +48,19 @@ jobs: VERBOSE: ${{ inputs.verbose && 'true' || '' }} run: | ROOT_DIR=$(realpath $PWD) + echo "::info ::checking stream-applications for apps" pushd stream-applications > /dev/null $ROOT_DIR/create-matrixes.sh + if [ ! -f matrix.json ]; then + echo "Expected to find matrix.json in:" + ls -al + exit 2 + fi COUNT=$(jq '.count' matrix.json) MAX_PARALLEL=$((5 * COUNT / 4)) if ((MAX_PARALLEL == COUNT)); then MAX_PARALLEL=$((COUNT + 1)) - fi + fi MATRIX=$(jq -c . matrix.json) PROCESSORS=$(jq -c '.processors' matrix.json) SINKS=$(jq -c '.sinks' matrix.json) @@ -60,11 +71,11 @@ jobs: echo "SINKS=$SINKS" >> $GITHUB_ENV echo "SOURCES=$SOURCES" >> $GITHUB_ENV if [ "${{ inputs.verbose }}" == "true" ]; then - echo "MAX_PARALLEL=$MAX_PARALLEL" - echo "MATRIX=$MATRIX" - echo "PROCESSORS=$PROCESSORS" - echo "SINKS=$SINKS" - echo "SOURCES=$SOURCES" + echo "::debug ::MAX_PARALLEL=$MAX_PARALLEL" + echo "::debug ::MATRIX=$MATRIX" + echo "::debug ::PROCESSORS=$PROCESSORS" + echo "::debug ::SINKS=$SINKS" + echo "::debug ::SOURCES=$SOURCES" fi popd if [ "${{ inputs.branch }}" == "main" ]; then @@ -82,110 +93,17 @@ jobs: matrix: ${{ env.matrix }} jdk_build: ${{ env.JDK_BUILD }} jre_version: ${{ env.JRE_DEFAULT }} - # scale-runners: - # runs-on: ubuntu-latest - # needs: - # - parameters - # concurrency: - # group: stream-apps-gh-runners - # cancel-in-progress: false - # steps: - # - name: 'Configure: checkout stream-applications' - # uses: actions/checkout@v3 - # with: - # ref: 'main' - # - name: Ensure scripts are executable - # shell: bash - # run: find . -type f -name "*.sh" -exec chmod a+x '{}' \; - # - name: 'Configure: Configure provider for stream-apps-gh-runners' - # shell: bash - # timeout-minutes: 15 - # run: | - # RUNNER_TYPE=$(./scripts/determine-provider.sh stream-apps-gh-runners) - # echo "RUNNER_TYPE=$RUNNER_TYPE" - # echo "RUNNER_TYPE=$RUNNER_TYPE" >> $GITHUB_ENV - # - name: 'Install: gcloud cli' - # if: ${{ env.RUNNER_TYPE == 'gke' }} - # uses: ./.github/actions/install-gcloud - # - name: 'Action: gcloud auth' - # if: ${{ env.RUNNER_TYPE == 'gke' }} - # id: auth_gcloud - # uses: 'google-github-actions/auth@v0' - # with: - # create_credentials_file: true - # credentials_json: ${{ secrets.GCP_CRED_JSON }} - # - name: 'Configure: Install TMC' - # if: ${{ env.RUNNER_TYPE == 'tmc' }} - # uses: ./.github/actions/install-tmc - # - name: 'Action: Login to TMC' - # if: ${{ env.RUNNER_TYPE == 'tmc' }} - # uses: ./.github/actions/auth-tmc - # timeout-minutes: 15 - # with: - # tmc_api_token: '${{ secrets.TMC_API_TOKEN }}' - # fail_on_error: false - # - name: 'Install: Groovy' - # uses: ./.github/actions/install-groovy - # with: - # version: 4.0.4 - # - name: 'Action: wait for cluster - stream-apps-gh-runners' - # shell: bash - # timeout-minutes: 5 - # env: - # VERBOSE: ${{ inputs.verbose && '--verbose' || '' }} - # run: ./scripts/wait-for-cluster-${RUNNER_TYPE}.sh stream-apps-gh-runners - # - name: 'Configure: Cluster Region' - # if: ${{ env.PROVIDER == 'gke' }} - # shell: bash - # run: | - # set +e - # REGION=$(gcloud container clusters list | grep -F "stream-apps-gh-runners" | awk '{print $2}') - # if [ "$REGION" == "" ]; then - # echo "CREATE_CLUSTER=true" >> $GITHUB_ENV - # else - # REG_MT=$(./scripts/determine-default.sh stream-apps-gh-runners "machine_type") - # export REGION - # CUR_MT=$(./scripts/determine-machine-type.sh) - # if [ "$REQ_MT" != "$CUR_MT" ]; then - # echo "::notice ::Current machinetype is $CUR_MT and required is $REQ_MT" - # echo "CREATE_CLUSTER=true" >> $GITHUB_ENV - # fi - # fi - # echo "REGION=${{ inputs.region }}" >> $GITHUB_ENV - # - name: 'Action: Re/Create SCDF PRO Runners' - # if: ${{ env.CREATE_CLUSTER == 'true' }} - # shell: bash - # env: - # VERBOSE: ${{ inputs.verbose && '--verbose' || '' }} - # run: | - # set +e - # ./scripts/delete-runners-${RUNNER_TYPE}.sh - # set -e - # ./scripts/create-runners-cluster-${RUNNER_TYPE}.sh - # - name: 'Action: scale cluster - stream-apps-gh-runners for ${{ needs.parameters.outputs.max_parallel }} runners' - # shell: bash - # timeout-minutes: 30 - # env: - # CLUSTER_NAME: 'stream-apps-gh-runners' - # VERBOSE: ${{ inputs.verbose && '--verbose' || '' }} - # run: | - # echo "::notice ::Scaling stream-apps-gh-runners to ${{ needs.parameters.outputs.max_parallel }} pods" - # ./scripts/scale-cluster-pods.sh stream-apps-gh-runners ${{ needs.parameters.outputs.max_parallel }} - # - name: 'Check: Wait for cluster nodes: stream-apps-gh-runners' - # shell: bash - # timeout-minutes: 25 - # env: - # VERBOSE: ${{ inputs.verbose && '--verbose' || '' }} - # run: | - # echo "::notice ::Waiting for cluster stream-apps-gh-runners and it's nodes" - # ./scripts/wait-for-cluster-${RUNNER_TYPE}.sh stream-apps-gh-runners --nodes - # - name: 'Action: Increase runners with ${{ needs.parameters.outputs.max_parallel }}' - # timeout-minutes: 10 - # uses: ./.github/actions/increase-runners - # with: - # inc: ${{ needs.parameters.outputs.max_parallel }} - # outputs: - # runner-type: '${{ env.RUNNER_TYPE }}' +# scale-runners: +# if: ${{ inputs.scale }} +# needs: +# - parameters +# concurrency: +# group: stream-apps-gh-runners +# cancel-in-progress: false +# uses: ./.github/workflows/scale-runners-up.yml +# with: +# verbose: ${{ inputs.verbose }} +# max_parallel: ${{ toJson(needs.parameters.outputs.max_parallel) }} core: runs-on: ubuntu-latest needs: @@ -234,7 +152,7 @@ jobs: fail-fast: false matrix: app: ${{ fromJson(needs.parameters.outputs.processors) }} - runs-on: ubuntu-latest + runs-on: ${{ inputs.scale && 'ubuntu-latest' || 'stream-ci' }} steps: - name: 'Configure: checkout stream-applications' uses: actions/checkout@v3 @@ -279,7 +197,7 @@ jobs: fail-fast: false matrix: app: ${{ fromJson(needs.parameters.outputs.sinks) }} - runs-on: ubuntu-latest + runs-on: ${{ inputs.scale && 'ubuntu-latest' || 'stream-ci' }} steps: - name: 'Configure: checkout stream-applications' uses: actions/checkout@v3 @@ -324,7 +242,7 @@ jobs: # will be removed when private runners are used. matrix: app: ${{ fromJson(needs.parameters.outputs.sources) }} - runs-on: ubuntu-latest + runs-on: ${{ inputs.scale && 'ubuntu-latest' || 'stream-ci' }} steps: - name: 'Configure: checkout stream-applications' uses: actions/checkout@v3 @@ -361,11 +279,10 @@ jobs: VERBOSE: ${{ inputs.verbose }} run: ./publish-app.sh "stream-applications/applications/source/${{ matrix.app }}" # scale-down-runners: -# if: ${{ success() }} -# runs-on: ubuntu-latest +# if: ${{ inputs.scale && success() }} # needs: # - parameters -# - scale-runners +# - scale-runners-up # - core # - processors # - sinks @@ -373,42 +290,7 @@ jobs: # concurrency: # group: stream-apps-gh-runners # cancel-in-progress: false -# steps: -# - name: 'Configure: Checkout' -# uses: actions/checkout@v3 -# with: -# ref: 'main' -# - name: 'Action: Ensure scripts are executable' -# shell: bash -# run: find . -type f -name "*.sh" -exec chmod a+x '{}' \; -# - name: 'Configure: Runners Cluster provider' -# shell: bash -# run: | -# RUNNER_TYPE=$(./scripts/determine-provider.sh stream-apps-gh-runners) -# if [ "$RUNNER_TYPE" != "gke" ]; then -# RUNNER_TYPE=tmc -# fi -# echo "RUNNER_TYPE=$RUNNER_TYPE" >> $GITHUB_ENV -# - name: 'Install: gcloud cli' -# if: ${{ env.RUNNER_TYPE == 'gke' }} -# uses: ./.github/actions/install-gcloud -# - name: 'Action: gcloud auth' -# if: ${{ env.RUNNER_TYPE == 'gke' }} -# id: auth_gcloud -# uses: 'google-github-actions/auth@v0' -# with: -# create_credentials_file: true -# credentials_json: ${{ secrets.GCP_CRED_JSON }} -# - name: 'Configure: Install TMC' -# if: ${{ env.RUNNER_TYPE == 'tmc' }} -# uses: ./.github/actions/install-tmc -# - name: 'Action: Login to TMC' -# if: ${{ env.RUNNER_TYPE == 'tmc' }} -# uses: ./.github/actions/auth-tmc -# with: -# tmc_api_token: '${{ secrets.TMC_API_TOKEN }}' -# - name: 'Action: Decrease runners with ${{ needs.parameters.outputs.max_parallel }}' -# timeout-minutes: 10 -# uses: ./.github/actions/decrease-runners -# with: -# dec: ${{ needs.parameters.outputs.max_parallel }} +# uses: ./.github/workflows/scale-runners-down.yml +# with: +# verbose: ${{ inputs.verbose }} +# max_parallel: ${{ toJson(needs.parameters.outputs.max_parallel) }} diff --git a/.github/workflows/scale-runners-down.yml b/.github/workflows/scale-runners-down.yml new file mode 100644 index 00000000..2807a63f --- /dev/null +++ b/.github/workflows/scale-runners-down.yml @@ -0,0 +1,63 @@ +name: 'scale-down' +description: 'Scale Down Private GitHub Runners' + +on: + workflow_call: + inputs: + verbose: + required: false + type: boolean + default: false + max_parallel: + type: number + required: true + secrets: + GCP_CRED_JSON: + required: false + TMC_API_TOKEN: + required: false +jobs: + scale-down: + runs-on: ubuntu-latest + concurrency: + group: stream-apps-gh-runners + cancel-in-progress: false + steps: + - name: 'Configure: Checkout' + uses: actions/checkout@v3 + with: + ref: 'main' + - name: 'Action: Ensure scripts are executable' + shell: bash + run: find . -type f -name "*.sh" -exec chmod a+x '{}' \; + - name: 'Configure: Runners Cluster provider' + shell: bash + run: | + RUNNER_TYPE=$(./scripts/determine-provider.sh stream-apps-gh-runners) + if [ "$RUNNER_TYPE" != "gke" ]; then + RUNNER_TYPE=tmc + fi + echo "RUNNER_TYPE=$RUNNER_TYPE" >> $GITHUB_ENV + - name: 'Install: gcloud cli' + if: ${{ env.RUNNER_TYPE == 'gke' }} + uses: ./.github/actions/install-gcloud + - name: 'Action: gcloud auth' + if: ${{ env.RUNNER_TYPE == 'gke' }} + id: auth_gcloud + uses: 'google-github-actions/auth@v0' + with: + create_credentials_file: true + credentials_json: ${{ secrets.GCP_CRED_JSON }} + - name: 'Configure: Install TMC' + if: ${{ env.RUNNER_TYPE == 'tmc' }} + uses: ./.github/actions/install-tmc + - name: 'Action: Login to TMC' + if: ${{ env.RUNNER_TYPE == 'tmc' }} + uses: ./.github/actions/auth-tmc + with: + tmc_api_token: '${{ secrets.TMC_API_TOKEN }}' + - name: 'Action: Decrease runners with ${{ inputs.max_parallel }}' + timeout-minutes: 10 + uses: ./.github/actions/decrease-runners + with: + dec: ${{ inputs.max_parallel }} diff --git a/.github/workflows/scale-runners-up.yml b/.github/workflows/scale-runners-up.yml new file mode 100644 index 00000000..a4c121a0 --- /dev/null +++ b/.github/workflows/scale-runners-up.yml @@ -0,0 +1,132 @@ +name: 'scale-up' +description: 'Scale Up Private GitHub Runners' + +on: + workflow_call: + inputs: + max_parallel: + type: number + required: true + verbose: + type: boolean + required: false + default: false + outputs: + runner-type: + value: ${{ jobs.scale-up.outputs.runner-type }} + secrets: + GCP_CRED_JSON: + required: false + TMC_API_TOKEN: + required: false + GH_ARC_APP_ID: + required: false + GH_ARC_INSTALLATION_ID: + required: false + GH_ARC_PRIVATE_KEY: + required: false + GH_ARC_PAT: + required: false +jobs: + scale-up: + runs-on: ubuntu-latest + concurrency: + group: stream-apps-gh-runners + cancel-in-progress: false + steps: + - name: 'Configure: checkout stream-applications' + uses: actions/checkout@v3 + with: + ref: 'main' + - name: Ensure scripts are executable + shell: bash + run: find . -type f -name "*.sh" -exec chmod a+x '{}' \; + - name: 'Configure: Configure provider for stream-apps-gh-runners' + shell: bash + timeout-minutes: 15 + run: | + RUNNER_TYPE=$(./scripts/determine-provider.sh stream-apps-gh-runners) + echo "RUNNER_TYPE=$RUNNER_TYPE" + echo "RUNNER_TYPE=$RUNNER_TYPE" >> $GITHUB_ENV + - name: 'Install: gcloud cli' + if: ${{ env.RUNNER_TYPE == 'gke' }} + uses: ./.github/actions/install-gcloud + - name: 'Action: gcloud auth' + if: ${{ env.RUNNER_TYPE == 'gke' }} + id: auth_gcloud + uses: 'google-github-actions/auth@v0' + with: + create_credentials_file: true + credentials_json: ${{ secrets.GCP_CRED_JSON }} + - name: 'Configure: Install TMC' + if: ${{ env.RUNNER_TYPE == 'tmc' }} + uses: ./.github/actions/install-tmc + - name: 'Action: Login to TMC' + if: ${{ env.RUNNER_TYPE == 'tmc' }} + uses: ./.github/actions/auth-tmc + timeout-minutes: 15 + with: + tmc_api_token: '${{ secrets.TMC_API_TOKEN }}' + fail_on_error: false + - name: 'Install: Groovy' + uses: ./.github/actions/install-groovy + with: + version: 4.0.4 + - name: 'Action: wait for cluster - stream-apps-gh-runners' + shell: bash + timeout-minutes: 5 + env: + VERBOSE: ${{ inputs.verbose && '--verbose' || '' }} + run: ./scripts/wait-for-cluster-${RUNNER_TYPE}.sh stream-apps-gh-runners + - name: 'Configure: Cluster Region' + if: ${{ env.PROVIDER == 'gke' }} + shell: bash + run: | + set +e + REGION=$(gcloud container clusters list | grep -F "stream-apps-gh-runners" | awk '{print $2}') + if [ "$REGION" == "" ]; then + echo "CREATE_CLUSTER=true" >> $GITHUB_ENV + else + REG_MT=$(./scripts/determine-default.sh stream-apps-gh-runners "machine_type") + export REGION + CUR_MT=$(./scripts/determine-machine-type.sh) + if [ "$REQ_MT" != "$CUR_MT" ]; then + echo "::notice ::Current machinetype is $CUR_MT and required is $REQ_MT" + echo "CREATE_CLUSTER=true" >> $GITHUB_ENV + fi + fi + echo "REGION=${{ inputs.region }}" >> $GITHUB_ENV + - name: 'Action: Re/Create SCDF PRO Runners' + if: ${{ env.CREATE_CLUSTER == 'true' }} + shell: bash + env: + VERBOSE: ${{ inputs.verbose && '--verbose' || '' }} + run: | + set +e + ./scripts/delete-runners-${RUNNER_TYPE}.sh + set -e + ./scripts/create-runners-cluster-${RUNNER_TYPE}.sh + - name: 'Action: scale cluster - stream-apps-gh-runners for ${{ inputs.max_parallel }} runners' + shell: bash + timeout-minutes: 30 + env: + CLUSTER_NAME: 'stream-apps-gh-runners' + VERBOSE: ${{ inputs.verbose && '--verbose' || '' }} + run: | + echo "::notice ::Scaling stream-apps-gh-runners to ${{ inputs.max_parallel }} pods" + ./scripts/scale-cluster-pods.sh stream-apps-gh-runners ${{ inputs.max_parallel }} + - name: 'Check: Wait for cluster nodes: stream-apps-gh-runners' + shell: bash + timeout-minutes: 25 + env: + VERBOSE: ${{ inputs.verbose && '--verbose' || '' }} + run: | + echo "::notice ::Waiting for cluster stream-apps-gh-runners and it's nodes" + ./scripts/wait-for-cluster-${RUNNER_TYPE}.sh stream-apps-gh-runners --nodes + - name: 'Action: Increase runners with ${{ inputs.max_parallel }}' + timeout-minutes: 10 + uses: ./.github/actions/increase-runners + with: + inc: ${{ inputs.max_parallel }} + outputs: + runner-type: '${{ env.RUNNER_TYPE }}' diff --git a/build-app.sh b/build-app.sh index 3c3f4fb6..721a479c 100755 --- a/build-app.sh +++ b/build-app.sh @@ -82,10 +82,10 @@ pushd $APP_FOLDER > /dev/null APPS=$(find * -maxdepth 0 -type d) for app in $APPS; do for v in $JDKS; do - echo "Pack:springcloudstream/$app:$VERSION-jdk$v" + echo "Pack:$app:$VERSION-jdk$v" set -e pack build \ - --path "springcloudstream/$app-$VERSION.jar" \ + --path "$app/target/$app-$VERSION.jar" \ --builder gcr.io/paketo-buildpacks/builder:base \ --env BP_JVM_VERSION=$v \ --env BPE_APPEND_JDK_JAVA_OPTIONS=-Dfile.encoding=UTF-8 \