[GitHub Actions] fix pack usage (#400)

.github/workflows/common.yml

@@ -13,6 +13,11 @@ on:
         required: false
         default: true
         description: 'Verbose output'
+      scale:
+        type: boolean
+        required: false
+        default: false
+        description: 'Scale and use private runners'
     secrets:
       DOCKER_HUB_USERNAME:
       DOCKER_HUB_PASSWORD:
@@ -43,13 +48,19 @@ jobs:
           VERBOSE: ${{ inputs.verbose && 'true' || '' }}
         run: |
           ROOT_DIR=$(realpath $PWD)
+          echo "::info ::checking stream-applications for apps"
           pushd stream-applications > /dev/null
             $ROOT_DIR/create-matrixes.sh
+            if [ ! -f matrix.json ]; then
+              echo "Expected to find matrix.json in:"
+              ls -al
+              exit 2
+            fi
             COUNT=$(jq '.count' matrix.json)
             MAX_PARALLEL=$((5 * COUNT / 4))
             if ((MAX_PARALLEL == COUNT)); then
               MAX_PARALLEL=$((COUNT + 1))
-            fi            
+            fi          
             MATRIX=$(jq -c . matrix.json)
             PROCESSORS=$(jq -c '.processors' matrix.json)
             SINKS=$(jq -c '.sinks' matrix.json)
@@ -60,11 +71,11 @@ jobs:
             echo "SINKS=$SINKS" >> $GITHUB_ENV
             echo "SOURCES=$SOURCES" >> $GITHUB_ENV
             if [ "${{ inputs.verbose }}" == "true" ]; then
-              echo "MAX_PARALLEL=$MAX_PARALLEL"
-              echo "MATRIX=$MATRIX"
-              echo "PROCESSORS=$PROCESSORS"
-              echo "SINKS=$SINKS"
-              echo "SOURCES=$SOURCES"
+              echo "::debug ::MAX_PARALLEL=$MAX_PARALLEL"
+              echo "::debug ::MATRIX=$MATRIX"
+              echo "::debug ::PROCESSORS=$PROCESSORS"
+              echo "::debug ::SINKS=$SINKS"
+              echo "::debug ::SOURCES=$SOURCES"
             fi
           popd
           if [ "${{ inputs.branch }}" == "main" ]; then
@@ -82,110 +93,17 @@ jobs:
       matrix: ${{ env.matrix }}
       jdk_build: ${{ env.JDK_BUILD }}
       jre_version: ${{ env.JRE_DEFAULT }}
-  #  scale-runners:
-  #    runs-on: ubuntu-latest
-  #    needs:
-  #      - parameters
-  #    concurrency:
-  #      group: stream-apps-gh-runners
-  #      cancel-in-progress: false
-  #    steps:
-  #      - name: 'Configure: checkout stream-applications'
-  #        uses: actions/checkout@v3
-  #        with:
-  #          ref: 'main'
-  #      - name: Ensure scripts are executable
-  #        shell: bash
-  #        run: find . -type f -name "*.sh" -exec chmod a+x '{}' \;
-  #      - name: 'Configure: Configure provider for stream-apps-gh-runners'
-  #        shell: bash
-  #        timeout-minutes: 15
-  #        run: |
-  #          RUNNER_TYPE=$(./scripts/determine-provider.sh stream-apps-gh-runners)
-  #          echo "RUNNER_TYPE=$RUNNER_TYPE"
-  #          echo "RUNNER_TYPE=$RUNNER_TYPE" >> $GITHUB_ENV
-  #      - name: 'Install: gcloud cli'
-  #        if: ${{ env.RUNNER_TYPE == 'gke' }}
-  #        uses: ./.github/actions/install-gcloud
-  #      - name: 'Action: gcloud auth'
-  #        if: ${{ env.RUNNER_TYPE == 'gke' }}
-  #        id: auth_gcloud
-  #        uses: 'google-github-actions/auth@v0'
-  #        with:
-  #          create_credentials_file: true
-  #          credentials_json: ${{ secrets.GCP_CRED_JSON }}
-  #      - name: 'Configure: Install TMC'
-  #        if: ${{ env.RUNNER_TYPE == 'tmc' }}
-  #        uses: ./.github/actions/install-tmc
-  #      - name: 'Action: Login to TMC'
-  #        if: ${{ env.RUNNER_TYPE == 'tmc' }}
-  #        uses: ./.github/actions/auth-tmc
-  #        timeout-minutes: 15
-  #        with:
-  #          tmc_api_token: '${{ secrets.TMC_API_TOKEN }}'
-  #          fail_on_error: false
-  #      - name: 'Install: Groovy'
-  #        uses: ./.github/actions/install-groovy
-  #        with:
-  #          version: 4.0.4
-  #      - name: 'Action: wait for cluster - stream-apps-gh-runners'
-  #        shell: bash
-  #        timeout-minutes: 5
-  #        env:
-  #          VERBOSE: ${{ inputs.verbose && '--verbose' || '' }}
-  #        run: ./scripts/wait-for-cluster-${RUNNER_TYPE}.sh stream-apps-gh-runners
-  #      - name: 'Configure: Cluster Region'
-  #        if: ${{ env.PROVIDER == 'gke' }}
-  #        shell: bash
-  #        run: |
-  #          set +e
-  #          REGION=$(gcloud container clusters list | grep -F "stream-apps-gh-runners" | awk '{print $2}')
-  #          if [ "$REGION" == "" ]; then
-  #            echo "CREATE_CLUSTER=true" >> $GITHUB_ENV
-  #          else
-  #            REG_MT=$(./scripts/determine-default.sh stream-apps-gh-runners "machine_type")
-  #            export REGION
-  #            CUR_MT=$(./scripts/determine-machine-type.sh)
-  #            if [ "$REQ_MT" != "$CUR_MT" ]; then
-  #              echo "::notice ::Current machinetype is $CUR_MT and required is $REQ_MT"
-  #              echo "CREATE_CLUSTER=true" >> $GITHUB_ENV
-  #            fi
-  #          fi
-  #          echo "REGION=${{ inputs.region }}" >> $GITHUB_ENV
-  #      - name: 'Action: Re/Create SCDF PRO Runners'
-  #        if: ${{ env.CREATE_CLUSTER == 'true' }}
-  #        shell: bash
-  #        env:
-  #          VERBOSE: ${{ inputs.verbose && '--verbose' || '' }}
-  #        run: |
-  #          set +e
-  #          ./scripts/delete-runners-${RUNNER_TYPE}.sh
-  #          set -e
-  #          ./scripts/create-runners-cluster-${RUNNER_TYPE}.sh
-  #      - name: 'Action: scale cluster - stream-apps-gh-runners for ${{ needs.parameters.outputs.max_parallel }} runners'
-  #        shell: bash
-  #        timeout-minutes: 30
-  #        env:
-  #          CLUSTER_NAME: 'stream-apps-gh-runners'
-  #          VERBOSE: ${{ inputs.verbose && '--verbose' || '' }}
-  #        run: |
-  #          echo "::notice ::Scaling stream-apps-gh-runners to ${{ needs.parameters.outputs.max_parallel }} pods"
-  #          ./scripts/scale-cluster-pods.sh stream-apps-gh-runners ${{ needs.parameters.outputs.max_parallel }}
-  #      - name: 'Check: Wait for cluster nodes: stream-apps-gh-runners'
-  #        shell: bash
-  #        timeout-minutes: 25
-  #        env:
-  #          VERBOSE: ${{ inputs.verbose && '--verbose' || '' }}
-  #        run: |
-  #          echo "::notice ::Waiting for cluster stream-apps-gh-runners and it's nodes"
-  #          ./scripts/wait-for-cluster-${RUNNER_TYPE}.sh stream-apps-gh-runners --nodes
-  #      - name: 'Action: Increase runners with ${{ needs.parameters.outputs.max_parallel }}'
-  #        timeout-minutes: 10
-  #        uses: ./.github/actions/increase-runners
-  #        with:
-  #          inc: ${{ needs.parameters.outputs.max_parallel }}
-  #    outputs:
-  #      runner-type: '${{ env.RUNNER_TYPE }}'
+#  scale-runners:
+#    if: ${{ inputs.scale }}
+#    needs:
+#      - parameters
+#    concurrency:
+#      group: stream-apps-gh-runners
+#      cancel-in-progress: false
+#    uses: ./.github/workflows/scale-runners-up.yml
+#    with:
+#      verbose: ${{ inputs.verbose }}
+#      max_parallel: ${{ toJson(needs.parameters.outputs.max_parallel) }}
   core:
     runs-on: ubuntu-latest
     needs:
@@ -234,7 +152,7 @@ jobs:
       fail-fast: false
       matrix:
         app: ${{ fromJson(needs.parameters.outputs.processors) }}
-    runs-on: ubuntu-latest
+    runs-on: ${{ inputs.scale && 'ubuntu-latest' || 'stream-ci' }}
     steps:
       - name: 'Configure: checkout stream-applications'
         uses: actions/checkout@v3
@@ -279,7 +197,7 @@ jobs:
       fail-fast: false
       matrix:
         app: ${{ fromJson(needs.parameters.outputs.sinks) }}
-    runs-on: ubuntu-latest
+    runs-on: ${{ inputs.scale && 'ubuntu-latest' || 'stream-ci' }}
     steps:
       - name: 'Configure: checkout stream-applications'
         uses: actions/checkout@v3
@@ -324,7 +242,7 @@ jobs:
       # will be removed when private runners are used.
       matrix:
         app: ${{ fromJson(needs.parameters.outputs.sources) }}
-    runs-on: ubuntu-latest
+    runs-on: ${{ inputs.scale && 'ubuntu-latest' || 'stream-ci' }}
     steps:
       - name: 'Configure: checkout stream-applications'
         uses: actions/checkout@v3
@@ -361,11 +279,10 @@ jobs:
           VERBOSE: ${{ inputs.verbose }}
         run: ./publish-app.sh "stream-applications/applications/source/${{ matrix.app }}"
 #  scale-down-runners:
-#    if: ${{ success() }}
-#    runs-on: ubuntu-latest
+#    if: ${{ inputs.scale && success() }}
 #    needs:
 #      - parameters
-#      - scale-runners
+#      - scale-runners-up
 #      - core
 #      - processors
 #      - sinks
@@ -373,42 +290,7 @@ jobs:
 #    concurrency:
 #      group: stream-apps-gh-runners
 #      cancel-in-progress: false
-#    steps:
-#      - name: 'Configure: Checkout'
-#        uses: actions/checkout@v3
-#        with:
-#          ref: 'main'
-#      - name: 'Action: Ensure scripts are executable'
-#        shell: bash
-#        run: find . -type f -name "*.sh" -exec chmod a+x '{}' \;
-#      - name: 'Configure: Runners Cluster provider'
-#        shell: bash
-#        run: |
-#          RUNNER_TYPE=$(./scripts/determine-provider.sh stream-apps-gh-runners)
-#          if [ "$RUNNER_TYPE" != "gke" ]; then
-#            RUNNER_TYPE=tmc
-#          fi
-#          echo "RUNNER_TYPE=$RUNNER_TYPE" >> $GITHUB_ENV
-#      - name: 'Install: gcloud cli'
-#        if: ${{ env.RUNNER_TYPE == 'gke' }}
-#        uses: ./.github/actions/install-gcloud
-#      - name: 'Action: gcloud auth'
-#        if: ${{ env.RUNNER_TYPE == 'gke' }}
-#        id: auth_gcloud
-#        uses: 'google-github-actions/auth@v0'
-#        with:
-#          create_credentials_file: true
-#          credentials_json: ${{ secrets.GCP_CRED_JSON }}
-#      - name: 'Configure: Install TMC'
-#        if: ${{ env.RUNNER_TYPE == 'tmc' }}
-#        uses: ./.github/actions/install-tmc
-#      - name: 'Action: Login to TMC'
-#        if: ${{ env.RUNNER_TYPE == 'tmc' }}
-#        uses: ./.github/actions/auth-tmc
-#        with:
-#          tmc_api_token: '${{ secrets.TMC_API_TOKEN }}'
-#      - name: 'Action: Decrease runners with ${{ needs.parameters.outputs.max_parallel }}'
-#        timeout-minutes: 10
-#        uses: ./.github/actions/decrease-runners
-#        with:
-#          dec: ${{ needs.parameters.outputs.max_parallel }}
+#    uses: ./.github/workflows/scale-runners-down.yml
+#    with:
+#      verbose: ${{ inputs.verbose }}
+#      max_parallel: ${{ toJson(needs.parameters.outputs.max_parallel) }}

.github/workflows/scale-runners-down.yml

@@ -0,0 +1,63 @@
+name: 'scale-down'
+description: 'Scale Down Private GitHub Runners'
+
+on:
+  workflow_call:
+    inputs:
+      verbose:
+        required: false
+        type: boolean
+        default: false
+      max_parallel:
+        type: number
+        required: true
+    secrets:
+      GCP_CRED_JSON:
+        required: false
+      TMC_API_TOKEN:
+        required: false
+jobs:
+  scale-down:
+    runs-on: ubuntu-latest
+    concurrency:
+      group: stream-apps-gh-runners
+      cancel-in-progress: false
+    steps:
+      - name: 'Configure: Checkout'
+        uses: actions/checkout@v3
+        with:
+          ref: 'main'
+      - name: 'Action: Ensure scripts are executable'
+        shell: bash
+        run: find . -type f -name "*.sh" -exec chmod a+x '{}' \;
+      - name: 'Configure: Runners Cluster provider'
+        shell: bash
+        run: |
+          RUNNER_TYPE=$(./scripts/determine-provider.sh stream-apps-gh-runners)
+          if [ "$RUNNER_TYPE" != "gke" ]; then
+            RUNNER_TYPE=tmc
+          fi
+          echo "RUNNER_TYPE=$RUNNER_TYPE" >> $GITHUB_ENV
+      - name: 'Install: gcloud cli'
+        if: ${{ env.RUNNER_TYPE == 'gke' }}
+        uses: ./.github/actions/install-gcloud
+      - name: 'Action: gcloud auth'
+        if: ${{ env.RUNNER_TYPE == 'gke' }}
+        id: auth_gcloud
+        uses: 'google-github-actions/auth@v0'
+        with:
+          create_credentials_file: true
+          credentials_json: ${{ secrets.GCP_CRED_JSON }}
+      - name: 'Configure: Install TMC'
+        if: ${{ env.RUNNER_TYPE == 'tmc' }}
+        uses: ./.github/actions/install-tmc
+      - name: 'Action: Login to TMC'
+        if: ${{ env.RUNNER_TYPE == 'tmc' }}
+        uses: ./.github/actions/auth-tmc
+        with:
+          tmc_api_token: '${{ secrets.TMC_API_TOKEN }}'
+      - name: 'Action: Decrease runners with ${{ inputs.max_parallel }}'
+        timeout-minutes: 10
+        uses: ./.github/actions/decrease-runners
+        with:
+          dec: ${{ inputs.max_parallel }}

.github/workflows/scale-runners-up.yml

@@ -0,0 +1,132 @@
+name: 'scale-up'
+description: 'Scale Up Private GitHub Runners'
+
+on:
+  workflow_call:
+    inputs:
+      max_parallel:
+        type: number
+        required: true
+      verbose:
+        type: boolean
+        required: false
+        default: false
+    outputs:
+      runner-type:
+        value: ${{ jobs.scale-up.outputs.runner-type }}
+    secrets:
+      GCP_CRED_JSON:
+        required: false
+      TMC_API_TOKEN:
+        required: false
+      GH_ARC_APP_ID:
+        required: false
+      GH_ARC_INSTALLATION_ID:
+        required: false
+      GH_ARC_PRIVATE_KEY:
+        required: false
+      GH_ARC_PAT:
+        required: false
+jobs:
+  scale-up:
+    runs-on: ubuntu-latest
+    concurrency:
+      group: stream-apps-gh-runners
+      cancel-in-progress: false
+    steps:
+      - name: 'Configure: checkout stream-applications'
+        uses: actions/checkout@v3
+        with:
+          ref: 'main'
+      - name: Ensure scripts are executable
+        shell: bash
+        run: find . -type f -name "*.sh" -exec chmod a+x '{}' \;
+      - name: 'Configure: Configure provider for stream-apps-gh-runners'
+        shell: bash
+        timeout-minutes: 15
+        run: |
+          RUNNER_TYPE=$(./scripts/determine-provider.sh stream-apps-gh-runners)
+          echo "RUNNER_TYPE=$RUNNER_TYPE"
+          echo "RUNNER_TYPE=$RUNNER_TYPE" >> $GITHUB_ENV
+      - name: 'Install: gcloud cli'
+        if: ${{ env.RUNNER_TYPE == 'gke' }}
+        uses: ./.github/actions/install-gcloud
+      - name: 'Action: gcloud auth'
+        if: ${{ env.RUNNER_TYPE == 'gke' }}
+        id: auth_gcloud
+        uses: 'google-github-actions/auth@v0'
+        with:
+          create_credentials_file: true
+          credentials_json: ${{ secrets.GCP_CRED_JSON }}
+      - name: 'Configure: Install TMC'
+        if: ${{ env.RUNNER_TYPE == 'tmc' }}
+        uses: ./.github/actions/install-tmc
+      - name: 'Action: Login to TMC'
+        if: ${{ env.RUNNER_TYPE == 'tmc' }}
+        uses: ./.github/actions/auth-tmc
+        timeout-minutes: 15
+        with:
+          tmc_api_token: '${{ secrets.TMC_API_TOKEN }}'
+          fail_on_error: false
+      - name: 'Install: Groovy'
+        uses: ./.github/actions/install-groovy
+        with:
+          version: 4.0.4
+      - name: 'Action: wait for cluster - stream-apps-gh-runners'
+        shell: bash
+        timeout-minutes: 5
+        env:
+          VERBOSE: ${{ inputs.verbose && '--verbose' || '' }}
+        run: ./scripts/wait-for-cluster-${RUNNER_TYPE}.sh stream-apps-gh-runners
+      - name: 'Configure: Cluster Region'
+        if: ${{ env.PROVIDER == 'gke' }}
+        shell: bash
+        run: |
+          set +e
+          REGION=$(gcloud container clusters list | grep -F "stream-apps-gh-runners" | awk '{print $2}')
+          if [ "$REGION" == "" ]; then
+            echo "CREATE_CLUSTER=true" >> $GITHUB_ENV
+          else
+            REG_MT=$(./scripts/determine-default.sh stream-apps-gh-runners "machine_type")
+            export REGION
+            CUR_MT=$(./scripts/determine-machine-type.sh)
+            if [ "$REQ_MT" != "$CUR_MT" ]; then
+              echo "::notice ::Current machinetype is $CUR_MT and required is $REQ_MT"
+              echo "CREATE_CLUSTER=true" >> $GITHUB_ENV
+            fi
+          fi
+          echo "REGION=${{ inputs.region }}" >> $GITHUB_ENV
+      - name: 'Action: Re/Create SCDF PRO Runners'
+        if: ${{ env.CREATE_CLUSTER == 'true' }}
+        shell: bash
+        env:
+          VERBOSE: ${{ inputs.verbose && '--verbose' || '' }}
+        run: |
+          set +e
+          ./scripts/delete-runners-${RUNNER_TYPE}.sh
+          set -e
+          ./scripts/create-runners-cluster-${RUNNER_TYPE}.sh
+      - name: 'Action: scale cluster - stream-apps-gh-runners for ${{ inputs.max_parallel }} runners'
+        shell: bash
+        timeout-minutes: 30
+        env:
+          CLUSTER_NAME: 'stream-apps-gh-runners'
+          VERBOSE: ${{ inputs.verbose && '--verbose' || '' }}
+        run: |
+          echo "::notice ::Scaling stream-apps-gh-runners to ${{ inputs.max_parallel }} pods"
+          ./scripts/scale-cluster-pods.sh stream-apps-gh-runners ${{ inputs.max_parallel }}
+      - name: 'Check: Wait for cluster nodes: stream-apps-gh-runners'
+        shell: bash
+        timeout-minutes: 25
+        env:
+          VERBOSE: ${{ inputs.verbose && '--verbose' || '' }}
+        run: |
+          echo "::notice ::Waiting for cluster stream-apps-gh-runners and it's nodes"
+          ./scripts/wait-for-cluster-${RUNNER_TYPE}.sh stream-apps-gh-runners --nodes
+      - name: 'Action: Increase runners with ${{ inputs.max_parallel }}'
+        timeout-minutes: 10
+        uses: ./.github/actions/increase-runners
+        with:
+          inc: ${{ inputs.max_parallel }}
+    outputs:
+      runner-type: '${{ env.RUNNER_TYPE }}'

build-app.sh

@@ -82,10 +82,10 @@ pushd $APP_FOLDER > /dev/null
     APPS=$(find * -maxdepth 0 -type d)
     for app in $APPS; do
       for v in $JDKS; do
-        echo "Pack:springcloudstream/$app:$VERSION-jdk$v"
+        echo "Pack:$app:$VERSION-jdk$v"
         set -e
         pack build \
-          --path "springcloudstream/$app-$VERSION.jar" \
+          --path "$app/target/$app-$VERSION.jar" \
           --builder gcr.io/paketo-buildpacks/builder:base \
           --env BP_JVM_VERSION=$v \
           --env BPE_APPEND_JDK_JAVA_OPTIONS=-Dfile.encoding=UTF-8 \