diff --git a/.github/workflows/common.yml b/.github/workflows/common.yml index 36938a42..ef470384 100644 --- a/.github/workflows/common.yml +++ b/.github/workflows/common.yml @@ -54,110 +54,110 @@ jobs: echo "::set-output name=sinks=$(jq '.sinks' matrix.json) echo "::set-output name=sources=$(jq '.sources' matrix.json) popd - scale-runners: - runs-on: ubuntu-latest - needs: - - parameters - concurrency: - group: stream-apps-gh-runners - cancel-in-progress: false - steps: - - name: 'Configure: checkout stream-applications' - uses: actions/checkout@v3 - with: - ref: 'main' - - name: Ensure scripts are executable - shell: bash - run: find . -type f -name "*.sh" -exec chmod a+x '{}' \; - - name: 'Configure: Configure provider for stream-apps-gh-runners' - shell: bash - timeout-minutes: 15 - run: | - RUNNER_TYPE=$(./scripts/determine-provider.sh stream-apps-gh-runners) - echo "RUNNER_TYPE=$RUNNER_TYPE" - echo "RUNNER_TYPE=$RUNNER_TYPE" >> $GITHUB_ENV - - name: 'Install: gcloud cli' - if: ${{ env.RUNNER_TYPE == 'gke' }} - uses: ./.github/actions/install-gcloud - - name: 'Action: gcloud auth' - if: ${{ env.RUNNER_TYPE == 'gke' }} - id: auth_gcloud - uses: 'google-github-actions/auth@v0' - with: - create_credentials_file: true - credentials_json: ${{ secrets.GCP_CRED_JSON }} - - name: 'Configure: Install TMC' - if: ${{ env.RUNNER_TYPE == 'tmc' }} - uses: ./.github/actions/install-tmc - - name: 'Action: Login to TMC' - if: ${{ env.RUNNER_TYPE == 'tmc' }} - uses: ./.github/actions/auth-tmc - timeout-minutes: 15 - with: - tmc_api_token: '${{ secrets.TMC_API_TOKEN }}' - fail_on_error: false - - name: 'Install: Groovy' - uses: ./.github/actions/install-groovy - with: - version: 4.0.4 - - name: 'Action: wait for cluster - stream-apps-gh-runners' - shell: bash - timeout-minutes: 5 - env: - VERBOSE: ${{ inputs.verbose && '--verbose' || '' }} - run: ./scripts/wait-for-cluster-${RUNNER_TYPE}.sh stream-apps-gh-runners - - name: 'Configure: Cluster Region' - if: ${{ env.PROVIDER == 'gke' }} - shell: bash - run: | - set +e - REGION=$(gcloud container clusters list | grep -F "stream-apps-gh-runners" | awk '{print $2}') - if [ "$REGION" == "" ]; then - echo "CREATE_CLUSTER=true" >> $GITHUB_ENV - else - REG_MT=$(./scripts/determine-default.sh stream-apps-gh-runners "machine_type") - export REGION - CUR_MT=$(./scripts/determine-machine-type.sh) - if [ "$REQ_MT" != "$CUR_MT" ]; then - echo "::notice ::Current machinetype is $CUR_MT and required is $REQ_MT" - echo "CREATE_CLUSTER=true" >> $GITHUB_ENV - fi - fi - echo "REGION=${{ inputs.region }}" >> $GITHUB_ENV - - name: 'Action: Re/Create SCDF PRO Runners' - if: ${{ env.CREATE_CLUSTER == 'true' }} - shell: bash - env: - VERBOSE: ${{ inputs.verbose && '--verbose' || '' }} - run: | - set +e - ./scripts/delete-runners-${RUNNER_TYPE}.sh - set -e - ./scripts/create-runners-cluster-${RUNNER_TYPE}.sh - - name: 'Action: scale cluster - stream-apps-gh-runners for ${{ needs.parameters.outputs.max_parallel }} runners' - shell: bash - timeout-minutes: 30 - env: - CLUSTER_NAME: 'stream-apps-gh-runners' - VERBOSE: ${{ inputs.verbose && '--verbose' || '' }} - run: | - echo "::notice ::Scaling stream-apps-gh-runners to ${{ needs.parameters.outputs.max_parallel }} pods" - ./scripts/scale-cluster-pods.sh stream-apps-gh-runners ${{ needs.parameters.outputs.max_parallel }} - - name: 'Check: Wait for cluster nodes: stream-apps-gh-runners' - shell: bash - timeout-minutes: 25 - env: - VERBOSE: ${{ inputs.verbose && '--verbose' || '' }} - run: | - echo "::notice ::Waiting for cluster stream-apps-gh-runners and it's nodes" - ./scripts/wait-for-cluster-${RUNNER_TYPE}.sh stream-apps-gh-runners --nodes - - name: 'Action: Increase runners with ${{ needs.parameters.outputs.max_parallel }}' - timeout-minutes: 10 - uses: ./.github/actions/increase-runners - with: - inc: ${{ needs.parameters.outputs.max_parallel }} - outputs: - runner-type: '${{ env.RUNNER_TYPE }}' +# scale-runners: +# runs-on: ubuntu-latest +# needs: +# - parameters +# concurrency: +# group: stream-apps-gh-runners +# cancel-in-progress: false +# steps: +# - name: 'Configure: checkout stream-applications' +# uses: actions/checkout@v3 +# with: +# ref: 'main' +# - name: Ensure scripts are executable +# shell: bash +# run: find . -type f -name "*.sh" -exec chmod a+x '{}' \; +# - name: 'Configure: Configure provider for stream-apps-gh-runners' +# shell: bash +# timeout-minutes: 15 +# run: | +# RUNNER_TYPE=$(./scripts/determine-provider.sh stream-apps-gh-runners) +# echo "RUNNER_TYPE=$RUNNER_TYPE" +# echo "RUNNER_TYPE=$RUNNER_TYPE" >> $GITHUB_ENV +# - name: 'Install: gcloud cli' +# if: ${{ env.RUNNER_TYPE == 'gke' }} +# uses: ./.github/actions/install-gcloud +# - name: 'Action: gcloud auth' +# if: ${{ env.RUNNER_TYPE == 'gke' }} +# id: auth_gcloud +# uses: 'google-github-actions/auth@v0' +# with: +# create_credentials_file: true +# credentials_json: ${{ secrets.GCP_CRED_JSON }} +# - name: 'Configure: Install TMC' +# if: ${{ env.RUNNER_TYPE == 'tmc' }} +# uses: ./.github/actions/install-tmc +# - name: 'Action: Login to TMC' +# if: ${{ env.RUNNER_TYPE == 'tmc' }} +# uses: ./.github/actions/auth-tmc +# timeout-minutes: 15 +# with: +# tmc_api_token: '${{ secrets.TMC_API_TOKEN }}' +# fail_on_error: false +# - name: 'Install: Groovy' +# uses: ./.github/actions/install-groovy +# with: +# version: 4.0.4 +# - name: 'Action: wait for cluster - stream-apps-gh-runners' +# shell: bash +# timeout-minutes: 5 +# env: +# VERBOSE: ${{ inputs.verbose && '--verbose' || '' }} +# run: ./scripts/wait-for-cluster-${RUNNER_TYPE}.sh stream-apps-gh-runners +# - name: 'Configure: Cluster Region' +# if: ${{ env.PROVIDER == 'gke' }} +# shell: bash +# run: | +# set +e +# REGION=$(gcloud container clusters list | grep -F "stream-apps-gh-runners" | awk '{print $2}') +# if [ "$REGION" == "" ]; then +# echo "CREATE_CLUSTER=true" >> $GITHUB_ENV +# else +# REG_MT=$(./scripts/determine-default.sh stream-apps-gh-runners "machine_type") +# export REGION +# CUR_MT=$(./scripts/determine-machine-type.sh) +# if [ "$REQ_MT" != "$CUR_MT" ]; then +# echo "::notice ::Current machinetype is $CUR_MT and required is $REQ_MT" +# echo "CREATE_CLUSTER=true" >> $GITHUB_ENV +# fi +# fi +# echo "REGION=${{ inputs.region }}" >> $GITHUB_ENV +# - name: 'Action: Re/Create SCDF PRO Runners' +# if: ${{ env.CREATE_CLUSTER == 'true' }} +# shell: bash +# env: +# VERBOSE: ${{ inputs.verbose && '--verbose' || '' }} +# run: | +# set +e +# ./scripts/delete-runners-${RUNNER_TYPE}.sh +# set -e +# ./scripts/create-runners-cluster-${RUNNER_TYPE}.sh +# - name: 'Action: scale cluster - stream-apps-gh-runners for ${{ needs.parameters.outputs.max_parallel }} runners' +# shell: bash +# timeout-minutes: 30 +# env: +# CLUSTER_NAME: 'stream-apps-gh-runners' +# VERBOSE: ${{ inputs.verbose && '--verbose' || '' }} +# run: | +# echo "::notice ::Scaling stream-apps-gh-runners to ${{ needs.parameters.outputs.max_parallel }} pods" +# ./scripts/scale-cluster-pods.sh stream-apps-gh-runners ${{ needs.parameters.outputs.max_parallel }} +# - name: 'Check: Wait for cluster nodes: stream-apps-gh-runners' +# shell: bash +# timeout-minutes: 25 +# env: +# VERBOSE: ${{ inputs.verbose && '--verbose' || '' }} +# run: | +# echo "::notice ::Waiting for cluster stream-apps-gh-runners and it's nodes" +# ./scripts/wait-for-cluster-${RUNNER_TYPE}.sh stream-apps-gh-runners --nodes +# - name: 'Action: Increase runners with ${{ needs.parameters.outputs.max_parallel }}' +# timeout-minutes: 10 +# uses: ./.github/actions/increase-runners +# with: +# inc: ${{ needs.parameters.outputs.max_parallel }} +# outputs: +# runner-type: '${{ env.RUNNER_TYPE }}' core: runs-on: ubuntu-latest steps: @@ -168,6 +168,10 @@ jobs: - name: 'Configure: Ensure scripts are executable' shell: bash run: find . -type f -name "*.sh" -exec chmod a+x '{}' \; + - name: 'Configure: Install Java' + uses: actions/setup-java@v1 + with: + java-version: 17 - name: 'Action: build initial dependencies' shell: bash env: @@ -190,7 +194,7 @@ jobs: strategy: matrix: app: ${{ fromJson(needs.parameters.outputs.processors) }} - runs-on: stream-ci + runs-on: ubuntu-latest steps: - name: 'Configure: checkout stream-applications' uses: actions/checkout@v3 @@ -204,6 +208,10 @@ jobs: - name: Ensure scripts are executable shell: bash run: find . -type f -name "*.sh" -exec chmod a+x '{}' \; + - name: 'Configure: Install Java' + uses: actions/setup-java@v1 + with: + java-version: 17 - name: 'Build: ${{ matrix.app }}' shell: bash env: @@ -220,7 +228,7 @@ jobs: strategy: matrix: app: ${{ fromJson(needs.parameters.outputs.sinks) }} - runs-on: stream-ci + runs-on: ubuntu-latest steps: - name: 'Configure: checkout stream-applications' uses: actions/checkout@v3 @@ -234,6 +242,10 @@ jobs: - name: Ensure scripts are executable shell: bash run: find . -type f -name "*.sh" -exec chmod a+x '{}' \; + - name: 'Configure: Install Java' + uses: actions/setup-java@v1 + with: + java-version: 17 - name: 'Build: ${{ matrix.app }}' shell: bash env: @@ -264,6 +276,10 @@ jobs: - name: Ensure scripts are executable shell: bash run: find . -type f -name "*.sh" -exec chmod a+x '{}' \; + - name: 'Configure: Install Java' + uses: actions/setup-java@v1 + with: + java-version: 17 - name: 'Build: ${{ matrix.app }}' shell: bash env: @@ -273,55 +289,55 @@ jobs: CI_DEPLOY_USERNAME: ${{ secrets.CI_DEPLOY_USERNAME }} CI_DEPLOY_PASSWORD: ${{ secrets.CI_DEPLOY_PASSWORD }} run: ./build-apps.sh "stream-applications/applications/source/${{ matrix.app }}" - scale-down-runners: - if: ${{ success() }} - runs-on: ubuntu-latest - needs: - - parameters - - scale-runners - - core - - processors - - sinks - - sources - concurrency: - group: stream-apps-gh-runners - cancel-in-progress: false - steps: - - name: 'Configure: Checkout' - uses: actions/checkout@v3 - with: - ref: 'main' - - name: 'Action: Ensure scripts are executable' - shell: bash - run: find . -type f -name "*.sh" -exec chmod a+x '{}' \; - - name: 'Configure: Runners Cluster provider' - shell: bash - run: | - RUNNER_TYPE=$(./scripts/determine-provider.sh stream-apps-gh-runners) - if [ "$RUNNER_TYPE" != "gke" ]; then - RUNNER_TYPE=tmc - fi - echo "RUNNER_TYPE=$RUNNER_TYPE" >> $GITHUB_ENV - - name: 'Install: gcloud cli' - if: ${{ env.RUNNER_TYPE == 'gke' }} - uses: ./.github/actions/install-gcloud - - name: 'Action: gcloud auth' - if: ${{ env.RUNNER_TYPE == 'gke' }} - id: auth_gcloud - uses: 'google-github-actions/auth@v0' - with: - create_credentials_file: true - credentials_json: ${{ secrets.GCP_CRED_JSON }} - - name: 'Configure: Install TMC' - if: ${{ env.RUNNER_TYPE == 'tmc' }} - uses: ./.github/actions/install-tmc - - name: 'Action: Login to TMC' - if: ${{ env.RUNNER_TYPE == 'tmc' }} - uses: ./.github/actions/auth-tmc - with: - tmc_api_token: '${{ secrets.TMC_API_TOKEN }}' - - name: 'Action: Decrease runners with ${{ needs.parameters.outputs.max_parallel }}' - timeout-minutes: 10 - uses: ./.github/actions/decrease-runners - with: - dec: ${{ needs.parameters.outputs.max_parallel }} +# scale-down-runners: +# if: ${{ success() }} +# runs-on: ubuntu-latest +# needs: +# - parameters +# - scale-runners +# - core +# - processors +# - sinks +# - sources +# concurrency: +# group: stream-apps-gh-runners +# cancel-in-progress: false +# steps: +# - name: 'Configure: Checkout' +# uses: actions/checkout@v3 +# with: +# ref: 'main' +# - name: 'Action: Ensure scripts are executable' +# shell: bash +# run: find . -type f -name "*.sh" -exec chmod a+x '{}' \; +# - name: 'Configure: Runners Cluster provider' +# shell: bash +# run: | +# RUNNER_TYPE=$(./scripts/determine-provider.sh stream-apps-gh-runners) +# if [ "$RUNNER_TYPE" != "gke" ]; then +# RUNNER_TYPE=tmc +# fi +# echo "RUNNER_TYPE=$RUNNER_TYPE" >> $GITHUB_ENV +# - name: 'Install: gcloud cli' +# if: ${{ env.RUNNER_TYPE == 'gke' }} +# uses: ./.github/actions/install-gcloud +# - name: 'Action: gcloud auth' +# if: ${{ env.RUNNER_TYPE == 'gke' }} +# id: auth_gcloud +# uses: 'google-github-actions/auth@v0' +# with: +# create_credentials_file: true +# credentials_json: ${{ secrets.GCP_CRED_JSON }} +# - name: 'Configure: Install TMC' +# if: ${{ env.RUNNER_TYPE == 'tmc' }} +# uses: ./.github/actions/install-tmc +# - name: 'Action: Login to TMC' +# if: ${{ env.RUNNER_TYPE == 'tmc' }} +# uses: ./.github/actions/auth-tmc +# with: +# tmc_api_token: '${{ secrets.TMC_API_TOKEN }}' +# - name: 'Action: Decrease runners with ${{ needs.parameters.outputs.max_parallel }}' +# timeout-minutes: 10 +# uses: ./.github/actions/decrease-runners +# with: +# dec: ${{ needs.parameters.outputs.max_parallel }}