From 043b2c45959601d7edfd93290381bc4256dc6df2 Mon Sep 17 00:00:00 2001 From: Corneil du Plessis Date: Thu, 6 Oct 2022 16:08:37 +0200 Subject: [PATCH] Adding initial GitHub Actions --- .github/actions/decrease-runners/action.yml | 20 + .github/actions/increase-runners/action.yml | 20 + .github/actions/install-gcloud/action.yml | 17 + .github/actions/install-groovy/action.yml | 26 ++ .github/actions/install-tmc/action.yml | 16 + .../actions/there_can_be_only_one/action.yml | 14 + .github/workflows/ci-2021-0-x.yml | 18 + .github/workflows/ci-2021-1-x.yml | 18 + .github/workflows/ci-main.yml | 18 + .github/workflows/common.yml | 327 +++++++++++++ .github/workflows/recreate-runners.yml | 138 ++++++ build-app.sh | 45 ++ config/defaults.json | 20 + config/k8s-versions.json | 29 ++ create-matrixes.sh | 48 ++ scripts/arc/values.yml | 17 + scripts/calculate-nodes.groovy | 437 ++++++++++++++++++ scripts/calculate-pod-sizes.groovy | 149 ++++++ scripts/check-runners.sh | 19 + scripts/create-cluster-tf-gke.sh | 40 ++ scripts/create-cluster-tmc.sh | 26 ++ scripts/create-runners-cluster-gke.sh | 19 + scripts/create-runners-cluster-tmc.sh | 8 + scripts/decrease-runners-tmc.sh | 17 + scripts/decrease-runners.sh | 93 ++++ scripts/delete-cluster-gke.sh | 120 +++++ scripts/delete-cluster-tf-gke.sh | 36 ++ scripts/delete-cluster-tmc.sh | 34 ++ scripts/delete-k8s-ns.sh | 59 +++ scripts/delete-namespaces.sh | 37 ++ scripts/delete-runners-gke.sh | 15 + scripts/delete-runners-tmc.sh | 14 + scripts/deploy-gh-runners-gke.sh | 13 + scripts/deploy-gh-runners-tmc.sh | 11 + scripts/deploy-gh-runners.sh | 70 +++ scripts/determine-default.sh | 54 +++ scripts/determine-machine-type.sh | 17 + scripts/determine-provider.sh | 30 ++ scripts/increase-runners.sh | 98 ++++ scripts/is_there_only_one.sh | 26 ++ scripts/k8s/pod-priorities.yaml | 7 + .../k8s/runners-stream-ci-auto-template.yaml | 41 ++ scripts/k8s/runners-stream-ci-auto.yaml | 41 ++ .../runners-stream-ci-manual-template.yaml | 24 + scripts/k8s/runners-stream-ci-manual.yaml | 24 + scripts/kubeconfig-gke.sh | 46 ++ scripts/kubeconfig-runners.sh | 13 + scripts/kubeconfig-tmc.sh | 29 ++ scripts/limit-runners.sh | 75 +++ scripts/list-clusters-gke.sh | 14 + scripts/list-clusters-tmc.sh | 14 + scripts/list-last-workflows.sh | 23 + scripts/list-workflow-runs.sh | 10 + scripts/report-pod-changes.sh | 32 ++ scripts/save_gcp_cred_json.sh | 18 + scripts/scale-cluster-nodes.sh | 188 ++++++++ scripts/scale-cluster-pods.sh | 45 ++ .../stream-apps-gh-runners/kube-admin.yaml | 19 + .../kubeconfig-template.yaml | 20 + .../terraform/stream-apps-gh-runners/main.tf | 172 +++++++ .../stream-apps-gh-runners/output.tf | 19 + .../stream-apps-gh-runners/terraform.tfvars | 10 + scripts/terraform/tf-apply.sh | 85 ++++ scripts/terraform/tf-destroy.sh | 44 ++ scripts/there_can_be_only_one.sh | 23 + scripts/update-cluster-tf-gke.sh | 33 ++ scripts/update-runner-values.sh | 6 + scripts/use-gke.sh | 33 ++ scripts/use-runners.sh | 14 + scripts/use-tmc.sh | 24 + scripts/wait-deployment.sh | 37 ++ scripts/wait-for-cluster-gke.sh | 57 +++ scripts/wait-for-cluster-tmc.sh | 41 ++ scripts/wait-k8s.sh | 38 ++ 74 files changed, 3552 insertions(+) create mode 100644 .github/actions/decrease-runners/action.yml create mode 100644 .github/actions/increase-runners/action.yml create mode 100644 .github/actions/install-gcloud/action.yml create mode 100644 .github/actions/install-groovy/action.yml create mode 100644 .github/actions/install-tmc/action.yml create mode 100644 .github/actions/there_can_be_only_one/action.yml create mode 100644 .github/workflows/ci-2021-0-x.yml create mode 100644 .github/workflows/ci-2021-1-x.yml create mode 100644 .github/workflows/ci-main.yml create mode 100644 .github/workflows/common.yml create mode 100644 .github/workflows/recreate-runners.yml create mode 100644 build-app.sh create mode 100644 config/defaults.json create mode 100644 config/k8s-versions.json create mode 100755 create-matrixes.sh create mode 100644 scripts/arc/values.yml create mode 100644 scripts/calculate-nodes.groovy create mode 100644 scripts/calculate-pod-sizes.groovy create mode 100755 scripts/check-runners.sh create mode 100755 scripts/create-cluster-tf-gke.sh create mode 100755 scripts/create-cluster-tmc.sh create mode 100755 scripts/create-runners-cluster-gke.sh create mode 100755 scripts/create-runners-cluster-tmc.sh create mode 100755 scripts/decrease-runners-tmc.sh create mode 100755 scripts/decrease-runners.sh create mode 100755 scripts/delete-cluster-gke.sh create mode 100755 scripts/delete-cluster-tf-gke.sh create mode 100755 scripts/delete-cluster-tmc.sh create mode 100755 scripts/delete-k8s-ns.sh create mode 100755 scripts/delete-namespaces.sh create mode 100755 scripts/delete-runners-gke.sh create mode 100755 scripts/delete-runners-tmc.sh create mode 100755 scripts/deploy-gh-runners-gke.sh create mode 100755 scripts/deploy-gh-runners-tmc.sh create mode 100755 scripts/deploy-gh-runners.sh create mode 100755 scripts/determine-default.sh create mode 100644 scripts/determine-machine-type.sh create mode 100755 scripts/determine-provider.sh create mode 100755 scripts/increase-runners.sh create mode 100755 scripts/is_there_only_one.sh create mode 100644 scripts/k8s/pod-priorities.yaml create mode 100644 scripts/k8s/runners-stream-ci-auto-template.yaml create mode 100644 scripts/k8s/runners-stream-ci-auto.yaml create mode 100644 scripts/k8s/runners-stream-ci-manual-template.yaml create mode 100644 scripts/k8s/runners-stream-ci-manual.yaml create mode 100755 scripts/kubeconfig-gke.sh create mode 100755 scripts/kubeconfig-runners.sh create mode 100755 scripts/kubeconfig-tmc.sh create mode 100755 scripts/limit-runners.sh create mode 100755 scripts/list-clusters-gke.sh create mode 100755 scripts/list-clusters-tmc.sh create mode 100755 scripts/list-last-workflows.sh create mode 100755 scripts/list-workflow-runs.sh create mode 100755 scripts/report-pod-changes.sh create mode 100755 scripts/save_gcp_cred_json.sh create mode 100755 scripts/scale-cluster-nodes.sh create mode 100755 scripts/scale-cluster-pods.sh create mode 100644 scripts/terraform/stream-apps-gh-runners/kube-admin.yaml create mode 100644 scripts/terraform/stream-apps-gh-runners/kubeconfig-template.yaml create mode 100644 scripts/terraform/stream-apps-gh-runners/main.tf create mode 100644 scripts/terraform/stream-apps-gh-runners/output.tf create mode 100644 scripts/terraform/stream-apps-gh-runners/terraform.tfvars create mode 100755 scripts/terraform/tf-apply.sh create mode 100755 scripts/terraform/tf-destroy.sh create mode 100755 scripts/there_can_be_only_one.sh create mode 100755 scripts/update-cluster-tf-gke.sh create mode 100755 scripts/update-runner-values.sh create mode 100755 scripts/use-gke.sh create mode 100755 scripts/use-runners.sh create mode 100755 scripts/use-tmc.sh create mode 100755 scripts/wait-deployment.sh create mode 100755 scripts/wait-for-cluster-gke.sh create mode 100755 scripts/wait-for-cluster-tmc.sh create mode 100755 scripts/wait-k8s.sh diff --git a/.github/actions/decrease-runners/action.yml b/.github/actions/decrease-runners/action.yml new file mode 100644 index 00000000..d976062a --- /dev/null +++ b/.github/actions/decrease-runners/action.yml @@ -0,0 +1,20 @@ +name: 'Decrease Runner Count' +description: 'Decrease the number of runners' +inputs: + dec: + description: 'number to decrease' + required: true + min: + description: 'minimum number of runners' + required: false +runs: + using: "composite" + steps: + - name: Install kubectl + uses: azure/setup-kubectl@v1 + - name: Decrease runners with ${{ inputs.dec }} + shell: bash + run: | + source ./scripts/kubeconfig-runners.sh + ./scripts/decrease-runners.sh ${{ inputs.dec }} ${{ inputs.min }} + echo "::notice ::Runners decreased with ${{ inputs.dec }}" diff --git a/.github/actions/increase-runners/action.yml b/.github/actions/increase-runners/action.yml new file mode 100644 index 00000000..e31234c4 --- /dev/null +++ b/.github/actions/increase-runners/action.yml @@ -0,0 +1,20 @@ +name: 'Increase Runner Count' +description: 'Increase the number of runners' +inputs: + inc: + description: 'number to increase' + required: true + max: + description: 'maximum allowed' + required: false +runs: + using: "composite" + steps: + - name: Install kubectl + uses: azure/setup-kubectl@v1 + - name: Increase runners with ${{ inputs.inc }} + shell: bash + run: | + source ./scripts/kubeconfig-runners.sh + ./scripts/increase-runners.sh ${{ inputs.inc }} ${{ inputs.max }} + echo "::notice ::Increased runners with ${{ inputs.inc }}" diff --git a/.github/actions/install-gcloud/action.yml b/.github/actions/install-gcloud/action.yml new file mode 100644 index 00000000..3dc2c3b3 --- /dev/null +++ b/.github/actions/install-gcloud/action.yml @@ -0,0 +1,17 @@ +name: Install GCP CLI +description: Install Google cloud CLI +inputs: + credentials_json: + required: true + description: 'GCP_CRED_JSON from secrets' +outputs: + credentials_file_path: + value: ${{ steps.google_auth.outputs.credentials_file_path }} + description: 'Path of GCP credentials json file' +runs: + using: "composite" + steps: + - name: 'Set up Cloud SDK' + uses: 'google-github-actions/setup-gcloud@v0' + with: + install_components: kubectl,docker-credential-gcr,gke-gcloud-auth-plugin \ No newline at end of file diff --git a/.github/actions/install-groovy/action.yml b/.github/actions/install-groovy/action.yml new file mode 100644 index 00000000..2059052e --- /dev/null +++ b/.github/actions/install-groovy/action.yml @@ -0,0 +1,26 @@ +name: 'Install Groovy using SDKMAN' +description: 'Install Groovy using SDKMAN' +inputs: + version: + required: false + description: 'Version number to use' +runs: + using: "composite" + steps: + - name: 'Download and install Groovy ${{ inputs.version }} using SDKMAN' + shell: bash + run: | + VERSION=${{ inputs.version }} + if [ "$VERSION" == "" ]; then + VERSION="4.0.4" + fi + ZIP="apache-groovy-binary-${VERSION}.zip" + curl -s -o "$ZIP" "https://groovy.jfrog.io/artifactory/dist-release-local/groovy-zips/apache-groovy-binary-${VERSION}.zip" + GROOVY_ROOT="$HOME/.groovy" + mkdir -p "$GROOVY_ROOT" + GROOVY_HOME="$GROOVY_ROOT/groovy-$VERSION" + unzip -u -q "$ZIP" -d "$GROOVY_ROOT" + echo "$GROOVY_HOME/bin" >> $GITHUB_PATH + export PATH="$PATH:$GROOVY_HOME/bin" + groovy -version + echo "::notice ::Groovy installed" \ No newline at end of file diff --git a/.github/actions/install-tmc/action.yml b/.github/actions/install-tmc/action.yml new file mode 100644 index 00000000..c2bf3f61 --- /dev/null +++ b/.github/actions/install-tmc/action.yml @@ -0,0 +1,16 @@ +name: 'Install TMC CLI' +description: 'Install TMC CLI' +runs: + using: "composite" + steps: + - name: 'Download and install TMC' + shell: bash + run: | + if [ ! -f /usr/local/bin/tmc ]; then + curl -sLO https://tmc-cli.s3-us-west-2.amazonaws.com/tmc/latest/linux/x64/tmc + chmod +x tmc + sudo mv tmc /usr/local/bin/ + echo "::notice ::TMC CLI installed" + else + echo "::notice ::TMC CLI already installed" + fi \ No newline at end of file diff --git a/.github/actions/there_can_be_only_one/action.yml b/.github/actions/there_can_be_only_one/action.yml new file mode 100644 index 00000000..963bb4d3 --- /dev/null +++ b/.github/actions/there_can_be_only_one/action.yml @@ -0,0 +1,14 @@ +name: 'There can be only one' +description: 'Checks that this is only current workflow running' +inputs: + github_token: + description: 'GITHUB_TOKEN' + required: true +runs: + using: "composite" + steps: + - name: 'Check active' + shell: bash + env: + GH_TOKEN: ${{ inputs.github_token }} + run: ./scripts/there_can_be_only_one.sh "${{ github.workflow }}" diff --git a/.github/workflows/ci-2021-0-x.yml b/.github/workflows/ci-2021-0-x.yml new file mode 100644 index 00000000..5174a0fc --- /dev/null +++ b/.github/workflows/ci-2021-0-x.yml @@ -0,0 +1,18 @@ +name: 'CI - 2021.0.x' + +on: + workflow_dispatch: + +jobs: + build: + uses: ./.github/workflows/common.yml + secrets: + DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} + DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} + TMC_API_TOKEN: ${{ secrets.TMC_API_TOKEN }} + GCP_CRED_JSON: ${{ secrets.GCP_CRED_JSON }} + CI_DEPLOY_USERNAME: ${{ secrets.CI_DEPLOY_USERNAME }} + CI_DEPLOY_PASSWORD: ${{ secrets.CI_DEPLOY_PASSWORD }} + with: + version: '2021.0.x' + branch: '2021.0.x' diff --git a/.github/workflows/ci-2021-1-x.yml b/.github/workflows/ci-2021-1-x.yml new file mode 100644 index 00000000..75397f92 --- /dev/null +++ b/.github/workflows/ci-2021-1-x.yml @@ -0,0 +1,18 @@ +name: 'CI - 2021.1.x' + +on: + workflow_dispatch: + +jobs: + build: + uses: ./.github/workflows/common.yml + secrets: + DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} + DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} + TMC_API_TOKEN: ${{ secrets.TMC_API_TOKEN }} + GCP_CRED_JSON: ${{ secrets.GCP_CRED_JSON }} + CI_DEPLOY_USERNAME: ${{ secrets.CI_DEPLOY_USERNAME }} + CI_DEPLOY_PASSWORD: ${{ secrets.CI_DEPLOY_PASSWORD }} + with: + version: '2021.1.x' + branch: '2021.1.x' diff --git a/.github/workflows/ci-main.yml b/.github/workflows/ci-main.yml new file mode 100644 index 00000000..31a48491 --- /dev/null +++ b/.github/workflows/ci-main.yml @@ -0,0 +1,18 @@ +name: 'CI - Main' + +on: + workflow_dispatch: + +jobs: + build: + uses: ./.github/workflows/common.yml + secrets: + DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} + DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} + TMC_API_TOKEN: ${{ secrets.TMC_API_TOKEN }} + GCP_CRED_JSON: ${{ secrets.GCP_CRED_JSON }} + CI_DEPLOY_USERNAME: ${{ secrets.CI_DEPLOY_USERNAME }} + CI_DEPLOY_PASSWORD: ${{ secrets.CI_DEPLOY_PASSWORD }} + with: + version: 'main' + branch: 'main' diff --git a/.github/workflows/common.yml b/.github/workflows/common.yml new file mode 100644 index 00000000..36938a42 --- /dev/null +++ b/.github/workflows/common.yml @@ -0,0 +1,327 @@ +name: 'Stream Applications - Common' + +on: + workflow_call: + inputs: + version: + type: string + required: false + default: 'main' + description: 'Version Tag' + branch: + type: string + required: false + default: 'main' + description: 'Version Tag' + secrets: + DOCKER_HUB_USERNAME: + DOCKER_HUB_PASSWORD: + TMC_API_TOKEN: + GCP_CRED_JSON: + CI_DEPLOY_USERNAME: + CI_DEPLOY_PASSWORD: + +jobs: + parameters: + runs-on: ubuntu-latest + steps: + - name: 'Configure: checkout' + uses: actions/checkout@v3 + with: + ref: 'main' + - name: 'Configure: checkout stream-applications@${{ inputs.branch }}' + uses: actions/checkout@v3 + with: + ref: ${{ inputs.branch }} + path: 'stream-applications' + - name: 'Configure: Ensure scripts are executable' + shell: bash + run: find . -type f -name "*.sh" -exec chmod a+x '{}' \; + - name: 'Configure: create streams applications matrix' + shell: bash + run: | + ROOT_DIR=$PWD + pushd stream-applications > /dev/null + $ROOT_DIR/create-matrixes.sh + COUNT=$(jq '.count' matrix.json) + MAX_PARALLEL=$((5 * COUNT / 4)) + if ((MAX_PARALLEL == COUNT)); then + MAX_PARALLEL=$((COUNT + 1)) + fi + echo "::set-output name=max_parallel=$MAX_PARALLEL" + echo "::set-output name=matrix=$(cat matrix.json) + echo "::set-output name=processors=$(jq '.processors' matrix.json) + echo "::set-output name=sinks=$(jq '.sinks' matrix.json) + echo "::set-output name=sources=$(jq '.sources' matrix.json) + popd + scale-runners: + runs-on: ubuntu-latest + needs: + - parameters + concurrency: + group: stream-apps-gh-runners + cancel-in-progress: false + steps: + - name: 'Configure: checkout stream-applications' + uses: actions/checkout@v3 + with: + ref: 'main' + - name: Ensure scripts are executable + shell: bash + run: find . -type f -name "*.sh" -exec chmod a+x '{}' \; + - name: 'Configure: Configure provider for stream-apps-gh-runners' + shell: bash + timeout-minutes: 15 + run: | + RUNNER_TYPE=$(./scripts/determine-provider.sh stream-apps-gh-runners) + echo "RUNNER_TYPE=$RUNNER_TYPE" + echo "RUNNER_TYPE=$RUNNER_TYPE" >> $GITHUB_ENV + - name: 'Install: gcloud cli' + if: ${{ env.RUNNER_TYPE == 'gke' }} + uses: ./.github/actions/install-gcloud + - name: 'Action: gcloud auth' + if: ${{ env.RUNNER_TYPE == 'gke' }} + id: auth_gcloud + uses: 'google-github-actions/auth@v0' + with: + create_credentials_file: true + credentials_json: ${{ secrets.GCP_CRED_JSON }} + - name: 'Configure: Install TMC' + if: ${{ env.RUNNER_TYPE == 'tmc' }} + uses: ./.github/actions/install-tmc + - name: 'Action: Login to TMC' + if: ${{ env.RUNNER_TYPE == 'tmc' }} + uses: ./.github/actions/auth-tmc + timeout-minutes: 15 + with: + tmc_api_token: '${{ secrets.TMC_API_TOKEN }}' + fail_on_error: false + - name: 'Install: Groovy' + uses: ./.github/actions/install-groovy + with: + version: 4.0.4 + - name: 'Action: wait for cluster - stream-apps-gh-runners' + shell: bash + timeout-minutes: 5 + env: + VERBOSE: ${{ inputs.verbose && '--verbose' || '' }} + run: ./scripts/wait-for-cluster-${RUNNER_TYPE}.sh stream-apps-gh-runners + - name: 'Configure: Cluster Region' + if: ${{ env.PROVIDER == 'gke' }} + shell: bash + run: | + set +e + REGION=$(gcloud container clusters list | grep -F "stream-apps-gh-runners" | awk '{print $2}') + if [ "$REGION" == "" ]; then + echo "CREATE_CLUSTER=true" >> $GITHUB_ENV + else + REG_MT=$(./scripts/determine-default.sh stream-apps-gh-runners "machine_type") + export REGION + CUR_MT=$(./scripts/determine-machine-type.sh) + if [ "$REQ_MT" != "$CUR_MT" ]; then + echo "::notice ::Current machinetype is $CUR_MT and required is $REQ_MT" + echo "CREATE_CLUSTER=true" >> $GITHUB_ENV + fi + fi + echo "REGION=${{ inputs.region }}" >> $GITHUB_ENV + - name: 'Action: Re/Create SCDF PRO Runners' + if: ${{ env.CREATE_CLUSTER == 'true' }} + shell: bash + env: + VERBOSE: ${{ inputs.verbose && '--verbose' || '' }} + run: | + set +e + ./scripts/delete-runners-${RUNNER_TYPE}.sh + set -e + ./scripts/create-runners-cluster-${RUNNER_TYPE}.sh + - name: 'Action: scale cluster - stream-apps-gh-runners for ${{ needs.parameters.outputs.max_parallel }} runners' + shell: bash + timeout-minutes: 30 + env: + CLUSTER_NAME: 'stream-apps-gh-runners' + VERBOSE: ${{ inputs.verbose && '--verbose' || '' }} + run: | + echo "::notice ::Scaling stream-apps-gh-runners to ${{ needs.parameters.outputs.max_parallel }} pods" + ./scripts/scale-cluster-pods.sh stream-apps-gh-runners ${{ needs.parameters.outputs.max_parallel }} + - name: 'Check: Wait for cluster nodes: stream-apps-gh-runners' + shell: bash + timeout-minutes: 25 + env: + VERBOSE: ${{ inputs.verbose && '--verbose' || '' }} + run: | + echo "::notice ::Waiting for cluster stream-apps-gh-runners and it's nodes" + ./scripts/wait-for-cluster-${RUNNER_TYPE}.sh stream-apps-gh-runners --nodes + - name: 'Action: Increase runners with ${{ needs.parameters.outputs.max_parallel }}' + timeout-minutes: 10 + uses: ./.github/actions/increase-runners + with: + inc: ${{ needs.parameters.outputs.max_parallel }} + outputs: + runner-type: '${{ env.RUNNER_TYPE }}' + core: + runs-on: ubuntu-latest + steps: + - name: 'Configure: checkout stream-applications@${{ inputs.branch }}' + uses: actions/checkout@v3 + with: + ref: ${{ inputs.branch }} + - name: 'Configure: Ensure scripts are executable' + shell: bash + run: find . -type f -name "*.sh" -exec chmod a+x '{}' \; + - name: 'Action: build initial dependencies' + shell: bash + env: + CI_DEPLOY_USERNAME: ${{ secrets.CI_DEPLOY_USERNAME }} + CI_DEPLOY_PASSWORD: ${{ secrets.CI_DEPLOY_PASSWORD }} + run: | + echo "::notice ::building - stream-applications-build" + ./mvnw deploy -f stream-applications-build -U + echo "::notice ::building - functions" + ./mvnw deploy -f functions -N -U + echo "::notice ::building - function-dependencies" + ./mvnw deploy -f functions/function-dependencies -N -U + echo "::notice ::building - stream-applications-core" + ./mvnw deploy -f applications/stream-applications-core -N -U + echo "::notice ::core build completed" + processors: + needs: + - core + - parameters + strategy: + matrix: + app: ${{ fromJson(needs.parameters.outputs.processors) }} + runs-on: stream-ci + steps: + - name: 'Configure: checkout stream-applications' + uses: actions/checkout@v3 + with: + ref: 'main' + - name: 'Configure: checkout stream-applications@${{ inputs.branch }}' + uses: actions/checkout@v3 + with: + ref: ${{ inputs.branch }} + path: 'stream-applications' + - name: Ensure scripts are executable + shell: bash + run: find . -type f -name "*.sh" -exec chmod a+x '{}' \; + - name: 'Build: ${{ matrix.app }}' + shell: bash + env: + VERSION: ${{ inputs.version }} + DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} + DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} + CI_DEPLOY_USERNAME: ${{ secrets.CI_DEPLOY_USERNAME }} + CI_DEPLOY_PASSWORD: ${{ secrets.CI_DEPLOY_PASSWORD }} + run: ./build-apps.sh "stream-applications/applications/processor/${{ matrix.app }}" + sinks: + needs: + - core + - parameters + strategy: + matrix: + app: ${{ fromJson(needs.parameters.outputs.sinks) }} + runs-on: stream-ci + steps: + - name: 'Configure: checkout stream-applications' + uses: actions/checkout@v3 + with: + ref: 'main' + - name: 'Configure: checkout stream-applications@${{ inputs.branch }}' + uses: actions/checkout@v3 + with: + ref: ${{ inputs.branch }} + path: 'stream-applications' + - name: Ensure scripts are executable + shell: bash + run: find . -type f -name "*.sh" -exec chmod a+x '{}' \; + - name: 'Build: ${{ matrix.app }}' + shell: bash + env: + VERSION: ${{ inputs.version }} + DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} + DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} + CI_DEPLOY_USERNAME: ${{ secrets.CI_DEPLOY_USERNAME }} + CI_DEPLOY_PASSWORD: ${{ secrets.CI_DEPLOY_PASSWORD }} + run: ./build-apps.sh "stream-applications/applications/sink/${{ matrix.app }}" + sources: + needs: + - core + - parameters + strategy: + matrix: + app: ${{ fromJson(needs.build.outputs.sources) }} + runs-on: ubuntu-latest + steps: + - name: 'Configure: checkout stream-applications' + uses: actions/checkout@v3 + with: + ref: 'main' + - name: 'Configure: checkout stream-applications@${{ inputs.branch }}' + uses: actions/checkout@v3 + with: + ref: ${{ inputs.branch }} + path: 'stream-applications' + - name: Ensure scripts are executable + shell: bash + run: find . -type f -name "*.sh" -exec chmod a+x '{}' \; + - name: 'Build: ${{ matrix.app }}' + shell: bash + env: + VERSION: ${{ inputs.version }} + DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} + DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} + CI_DEPLOY_USERNAME: ${{ secrets.CI_DEPLOY_USERNAME }} + CI_DEPLOY_PASSWORD: ${{ secrets.CI_DEPLOY_PASSWORD }} + run: ./build-apps.sh "stream-applications/applications/source/${{ matrix.app }}" + scale-down-runners: + if: ${{ success() }} + runs-on: ubuntu-latest + needs: + - parameters + - scale-runners + - core + - processors + - sinks + - sources + concurrency: + group: stream-apps-gh-runners + cancel-in-progress: false + steps: + - name: 'Configure: Checkout' + uses: actions/checkout@v3 + with: + ref: 'main' + - name: 'Action: Ensure scripts are executable' + shell: bash + run: find . -type f -name "*.sh" -exec chmod a+x '{}' \; + - name: 'Configure: Runners Cluster provider' + shell: bash + run: | + RUNNER_TYPE=$(./scripts/determine-provider.sh stream-apps-gh-runners) + if [ "$RUNNER_TYPE" != "gke" ]; then + RUNNER_TYPE=tmc + fi + echo "RUNNER_TYPE=$RUNNER_TYPE" >> $GITHUB_ENV + - name: 'Install: gcloud cli' + if: ${{ env.RUNNER_TYPE == 'gke' }} + uses: ./.github/actions/install-gcloud + - name: 'Action: gcloud auth' + if: ${{ env.RUNNER_TYPE == 'gke' }} + id: auth_gcloud + uses: 'google-github-actions/auth@v0' + with: + create_credentials_file: true + credentials_json: ${{ secrets.GCP_CRED_JSON }} + - name: 'Configure: Install TMC' + if: ${{ env.RUNNER_TYPE == 'tmc' }} + uses: ./.github/actions/install-tmc + - name: 'Action: Login to TMC' + if: ${{ env.RUNNER_TYPE == 'tmc' }} + uses: ./.github/actions/auth-tmc + with: + tmc_api_token: '${{ secrets.TMC_API_TOKEN }}' + - name: 'Action: Decrease runners with ${{ needs.parameters.outputs.max_parallel }}' + timeout-minutes: 10 + uses: ./.github/actions/decrease-runners + with: + dec: ${{ needs.parameters.outputs.max_parallel }} diff --git a/.github/workflows/recreate-runners.yml b/.github/workflows/recreate-runners.yml new file mode 100644 index 00000000..b1271d6c --- /dev/null +++ b/.github/workflows/recreate-runners.yml @@ -0,0 +1,138 @@ +name: 'Recreate Runners' + +on: + workflow_call: + inputs: + github_personal_access_token: + type: string + required: false + verbose: + type: string + required: false + default: 'false' + description: 'Increase verbosity of scripts' + secrets: + TMC_API_TOKEN: + required: true + GCP_CRED_JSON: + required: true + DOCKER_HUB_USERNAME: + required: true + DOCKER_HUB_PASSWORD: + required: true + GH_ARC_APP_ID: + required: false + GH_ARC_INSTALLATION_ID: + required: false + GH_ARC_PRIVATE_KEY: + required: false + GH_ARC_PAT: + required: false + workflow_dispatch: + inputs: + github_personal_access_token: + description: 'Github PAT for scdf bot account. If not provided will use GH_ARC_APP_ID' + required: false + verbose: + required: false + description: 'Increase verbosity of scripts' + default: 'false' +jobs: + recreate-runners: + runs-on: ubuntu-latest + concurrency: + group: stream-apps-gh-runners + cancel-in-progress: false + steps: + - name: 'Configure: checkout scdf-pro' + uses: actions/checkout@v3 + - name: 'Configure: ensure scripts are executable' + shell: bash + run: find . -type f -name "*.sh" -exec chmod a+x '{}' \; + - name: 'Check: active workflows' + uses: ./.github/actions/there_can_be_only_one + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + - name: 'Configure: runner type and cluster name' + shell: bash + run: | + echo "RUNNER_TYPE=$(./scripts/determine-provider.sh stream-apps-gh-runners)" >> $GITHUB_ENV + echo "CLUSTER_NAME=stream-apps-gh-runners" >> $GITHUB_ENV + - name: 'Configure: install TMC' + if: ${{ env.RUNNER_TYPE == 'tmc' }} + uses: ./.github/actions/install-tmc + - name: 'Configure: login to TMC' + if: ${{ env.RUNNER_TYPE == 'tmc' }} + uses: ./.github/actions/auth-tmc + with: + tmc_api_token: ${{ secrets.TMC_API_TOKEN }} + - name: 'Configure: install kubectl' + if: ${{ env.RUNNER_TYPE == 'tmc' }} + uses: azure/setup-kubectl@v1 + - name: 'Configure: install carvel' + if: ${{ env.RUNNER_TYPE == 'gke' }} + uses: vmware-tanzu/carvel-setup-action@v1 + with: + token: ${{ secrets.GITHUB_TOKEN }} + - name: 'Configure: install helm' + uses: azure/setup-helm@v3 + with: + token: ${{ secrets.GITHUB_TOKEN }} + - name: 'Configure: install gcloud cli' + if: ${{ env.RUNNER_TYPE == 'gke' }} + uses: ./.github/actions/auth-gcloud + - name: 'Action: gcloud auth' + if: ${{ env.RUNNER_TYPE == 'gke' }} + id: auth_gcloud + uses: 'google-github-actions/auth@v0' + with: + create_credentials_file: true + credentials_json: ${{ secrets.GCP_CRED_JSON }} + - name: 'Configure: gcp credentials filename' + if: ${{ env.RUNNER_TYPE == 'gke' }} + shell: bash + run: echo "GCP_CRED_JSON_FILE=${{ steps.auth_gcloud.outputs.credentials_file_path }}" >> $GITHUB_ENV + - name: 'Configure: install Terraform' + if: ${{ env.RUNNER_TYPE == 'gke' }} + uses: hashicorp/setup-terraform@v2 + - name: 'Action: delete GH Runners Cluster' + shell: bash + timeout-minutes: 10 + env: + VERBOSE: ${{ inputs.verbose == 'true' && '--verbose' || '' }} + run: | + set +e + source ./scripts/use-${RUNNER_TYPE}.sh $CLUSTER_NAME + RC=$? + if (( $RC == 0 )); then + set +e + ./scripts/delete-runners-${RUNNER_TYPE}.sh + fi + - name: 'Action: create GH Runners Cluster' + shell: bash + timeout-minutes: 30 + env: + VERBOSE: ${{ inputs.verbose == 'true' && '--verbose' || '' }} + run: ./scripts/create-runners-cluster-${RUNNER_TYPE}.sh + - name: 'Wait: for cluster' + shell: bash + timeout-minutes: 25 + env: + VERBOSE: ${{ inputs.verbose == 'true' && '--verbose' || '' }} + run: ./scripts/wait-for-cluster-${RUNNER_TYPE}.sh stream-apps-gh-runners + - name: 'Configure: kubectl for stream-apps-gh-runners' + shell: bash + env: + VERBOSE: ${{ inputs.verbose == 'true' && '--verbose' || '' }} + run: source ./scripts/kubeconfig-${RUNNER_TYPE}.sh stream-apps-gh-runners + - name: 'Configure: deploy actions-runner-controller into stream-apps-gh-runners' + shell: bash + env: + VERBOSE: ${{ inputs.verbose == 'true' && '--verbose' || '' }} + DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} + DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} + GH_ARC_APP_ID: ${{ secrets.GH_ARC_APP_ID }} + GH_ARC_INSTALLATION_ID: ${{ secrets.GH_ARC_INSTALLATION_ID }} + GH_ARC_PRIVATE_KEY: ${{ secrets.GH_ARC_PRIVATE_KEY }} + GH_ARC_PAT: ${{ inputs.github_personal_access_token }} + run: ./scripts/deploy-gh-runners-${RUNNER_TYPE}.sh diff --git a/build-app.sh b/build-app.sh new file mode 100644 index 00000000..2d3f941a --- /dev/null +++ b/build-app.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +(return 0 2>/dev/null) && sourced=1 || sourced=0 +function check_env() { + eval ev='$'$1 + if [ "$ev" == "" ]; then + echo "$1 not defined" + if (( sourced != 0 )); then + return 1 + else + exit 1 + fi + fi +} +if [ "$1" == "" ]; then + echo "Application folder required" + if((sourced > 0)); then + exit 0 + else + exit 1 + fi +fi +APP_FOLDER=$1 + +check_env DOCKER_HUB_USERNAME +check_env DOCKER_HUB_PASSWORD +check_env VERSION + +ROOT_DIR=$PWD + +pushd $APP_FOLDER > /dev/null + $ROOT_DIR/mvnw clean install + rm -rf apps + if [ -d "src/main/java" ]; then + $ROOT_DIR/mvnw deploy -U -Pintegration + else + $ROOT_DIR/mvnw package -U -Pintegration + fi + pushd apps + $ROOT_DIR/mvnw package jib:build -DskipTests \ + -Djib.to.tags="$VERSION" \ + -Djib.httpTimeout=1800000 \ + -Djib.to.auth.username="$DOCKER_HUB_USERNAME" \ + -Djib.to.auth.password="$DOCKER_HUB_PASSWORD" + popd +popd diff --git a/config/defaults.json b/config/defaults.json new file mode 100644 index 00000000..645296f8 --- /dev/null +++ b/config/defaults.json @@ -0,0 +1,20 @@ +{ + "default": { + "provider": "gke", + "region": "us-central1", + "k8s_version": "1.23", + "pods_per_job": 1, + "ram_per_pod": 4, + "cpu_per_pod": 1, + "scale_down": 0, + "disk_size": "200" + }, + "streams_apps_gh_runners": { + "pods_per_job": 1, + "ram_per_pod": 4, + "cpu_per_pod": 1, + "machine_type": "n2-standard-8", + "runner_scaling": "manual", + "scale_down": 1 + } +} diff --git a/config/k8s-versions.json b/config/k8s-versions.json new file mode 100644 index 00000000..e14ded26 --- /dev/null +++ b/config/k8s-versions.json @@ -0,0 +1,29 @@ +{ + "default_version": "1.23", + "versions": [ + { + "k8s": "1.19", + "k8s_version": "1.19.16-gke.15700" + }, + { + "k8s": "1.20", + "k8s_version": "1.20.15-gke.9900" + }, + { + "k8s": "1.21", + "k8s_version": "1.21.14-gke.2700" + }, + { + "k8s": "1.22", + "k8s_version": "1.22.12-gke.500" + }, + { + "k8s": "1.23", + "k8s_version": "1.23.8-gke.1900" + }, + { + "k8s": "1.24", + "k8s_version": "1.24.3-gke.2100" + } + ] +} diff --git a/create-matrixes.sh b/create-matrixes.sh new file mode 100755 index 00000000..a061ccea --- /dev/null +++ b/create-matrixes.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +pushd applications/processor > /dev/null +PROCESSORS=$(find * -maxdepth 0 -type d) +popd > /dev/null +pushd applications/sink > /dev/null +SINKS=$(find * -maxdepth 0 -type d) +popd > /dev/null +pushd applications/source > /dev/null +SOURCES=$(find * -maxdepth 0 -type d) +popd > /dev/null + +TOTAL=0 +echo "{" > matrix.json +echo "\"processors\":[" >> matrix.json +COUNT=0 +for app in $PROCESSORS; do + if ((COUNT > 0)); then + echo "," >> matrix.json + fi + echo "\"$app\"" >> matrix.json + COUNT=$((COUNT+1)) + TOTAL=$((TOTAL+1)) +done +echo "], \"sinks\":[" >> matrix.json +COUNT=0 +for app in $SINKS; do + if ((COUNT > 0)); then + echo "," >> matrix.json + fi + echo "\"$app\"" >> matrix.json + COUNT=$((COUNT+1)) + TOTAL=$((TOTAL+1)) +done +echo "], \"sources\":[" >> matrix.json +COUNT=0 +for app in $SOURCES; do + if ((COUNT > 0)); then + echo "," >> matrix.json + fi + echo "\"$app\"" >> matrix.json + COUNT=$((COUNT+1)) + TOTAL=$((TOTAL+1)) +done +echo "]" >> matrix.json +echo ",\"count\": $TOTAL" >> matrix.json +echo "}" >> matrix.json +MATRIX=$(jq -c . matrix.json) +echo "$MATRIX" > matrix.json diff --git a/scripts/arc/values.yml b/scripts/arc/values.yml new file mode 100644 index 00000000..cc072809 --- /dev/null +++ b/scripts/arc/values.yml @@ -0,0 +1,17 @@ +syncPeriod: 2m +authSecret: + create: false + name: controller-manager-secret +priorityClassName: +imagePullSecrets: + - name: scdf-metadata-default +githubWebhookServer: + priorityClassName: high-priority + secret: + name: controller-manager-secret + enabled: true +image: + tag: v0.26.0 +#runner: +# statusUpdateHook: +# enabled: true diff --git a/scripts/calculate-nodes.groovy b/scripts/calculate-nodes.groovy new file mode 100644 index 00000000..6cca0b1e --- /dev/null +++ b/scripts/calculate-nodes.groovy @@ -0,0 +1,437 @@ +import groovy.json.JsonOutput +import groovy.json.JsonSlurper + +import java.util.stream.Collectors + +double stringToGb(boolean verbose, String memory) { + double result = 0.5 + if (memory.endsWith('Mi')) { + result = Double.parseDouble(memory - 'Mi') / 1024.0 + } else if (memory.endsWith('Mb')) { + result = Double.parseDouble(memory - 'Mb') / 1000.0 + } else if (memory.endsWith('Gi')) { + result = Double.parseDouble(memory - 'Gi') + } else if (memory.endsWith('Gb')) { + result = Double.parseDouble(memory - 'Gb') + } else if (memory.endsWith('Ki')) { + result = Double.parseDouble(memory - 'Ki') / (1024.0 * 1024.0) + } else if (memory.endsWith('Kb')) { + result = Double.parseDouble(memory - 'Kb') / 1000000.0 + } + if (verbose) { + println "$memory=$result G" + } + return result +} + +double stringToCpu(boolean verbose, String cpu) { + double result = 0.1 + if (cpu.endsWith('m')) { + result = Double.parseDouble(cpu - 'm') / 1000.0 + } else { + result = Double.parseDouble(cpu) + } + if (verbose) { + println "$cpu=$result" + } + return result +} + +double memoryCalc(boolean verbose, String limit, String request) { + if (request != null) { + return stringToGb(verbose, request) + } + if (limit != null) { + return stringToGb(verbose, limit) + } + return 0.5 +} + +double cpuCalc(boolean verbose, String limit, String request) { + if (request != null) { + return stringToCpu(verbose, request) + } + if (limit != null) { + return stringToCpu(verbose, limit) + } + return 0.1 +} + +double calculateUsing(boolean verbose, Map matrix, Closure calc) { + return matrix.items.stream().filter { pod -> + if (verbose) { + println "Pod:$pod.metadata" + } + pod.status.containerStatuses.stream().anyMatch { containerStatus -> + boolean running = containerStatus.state?.running != null + if (verbose) { + println "Pod:${pod.metadata.name}:$running" + } + return running + } + }.map { pod -> + double m = pod.spec.containers.stream().map { container -> + double result = calc.call(pod, container) + if (verbose) { + println "Pod:${pod.metadata.name}:Container:${container.name}=$result" + } + return result + }.collect(Collectors.summingDouble(Double::doubleValue)) + if (verbose) { + println "Pod:${pod.metadata.name}=$m" + } + return m + }.collect(Collectors.summingDouble(Double::doubleValue)) +} + +Map calculatePodSizes(boolean verbose, String fileName) { + def file = new File(fileName) + if (!file.exists()) { + System.err.println "File not found: $file.absoluteFile" + System.exit(2) + } + JsonSlurper jsonSlurper = new JsonSlurper() + if (verbose) { + println "Loading: $file.absoluteFile" + } + def matrix = jsonSlurper.parse(file) + double memory = calculateUsing(verbose, matrix) { pod, container -> + double result = memoryCalc(verbose, container.resources.limits?.memory, container.resources.requests?.memory) + if (verbose) { + println "Pod:${pod.metadata.name}:Container:${container.name}=$result" + } + return result + } + double cpu = calculateUsing(verbose, matrix) { pod, container -> + double result = cpuCalc(verbose, container.resources.limits?.cpu, container.resources.requests?.cpu) + if (verbose) { + println "Pod:${pod.metadata.name}:Container:${container.name}=$result" + } + return result + } + return ['memory': memory, 'cpu': cpu] +} + +int divideRoundUp(int a, int b) { + int r = a / b + if (a % b != 0) { + r += 1 + } + return r +} + +void usageExit(String message = null) { + if (message != null) { + println message + } + println 'Usage: [--output ] [--verbose] [--shrink] [--machine ] [--current ] [--ram ] [--cpu ] [--min ] [--max ] [--nodes ] [--podfile ]' + println ''' +\tmachine-type: The machine used by provider for nodes. +\tcurrentNodeCount: Total number of nodes in cluster. +\toutputFile: JSON output { "nodes": number, "shrink": boolean } will be written to the file. +\tpodfile: Output of "kubectl get pods --A -o json" will be used to determine used cpus and memory. +\trequiredNodes: Total number of nodes required. If podFile is provided the available will be considered. +\trequiredRam: RAM required. machineType is required to calculate nodes. +\trequiredCpu: CPU required. machineType is required to calculate nodes. +\tmaximumNodes: Maximum number of nodes allowed in the cluster. +\tminimumNodes: Minimum number of nodes allowed in the cluster. +\tverbose: Provide extra output. +\tshrink: Allow a target lower than the currentNodeCount. +''' + System.exit(1) +} + +int currentNodes = 0 +int maxNodes = -1 +int minNodes = -1 +String outputFile +String machineType +String podFile; +int ramUsed = 0; +int cpuUsed = 0; + +int requiredRam = 0 +int requiredCpu = 0 +Integer requiredNodes = null +boolean shrink = false +boolean verbose = false +for (int i = 0; i < args.length; i++) { + String arg = args[i] + switch (arg) { + case '--shrink': + shrink = true + break + case '--verbose': + verbose = true + println "Arguments: $args" + break + case '--podfile': + if (args.length <= i + 1) { + usageExit('Missing arguments after --podfile') + } + podFile = args[i + 1] + i += 1 + break + case '--output': + if (args.length <= i + 1) { + usageExit('Missing arguments after --output') + } + outputFile = args[i + 1] + i += 1 + break + case '--cpu': + if (args.length <= i + 1) { + usageExit('Missing arguments after --cpu') + } + requiredCpu = args[i + 1].toInteger() + if (requiredCpu < 0) { + shrink = true + } + i += 1 + break + case '--ram': + if (args.length <= i + 1) { + usageExit('Missing arguments after --ram') + } + requiredRam = args[i + 1].toInteger() + if (requiredRam < 0) { + shrink = true + } + i += 1 + break + case '--nodes': + if (args.length <= i + 1) { + usageExit('Missing arguments after --nodes') + } + requiredNodes = args[i + 1].toInteger() + if (requiredNodes < 0) { + shrink = true + } + i += 1 + break + case '--max': + if (args.length <= i + 1) { + usageExit('Missing arguments after --max') + } + maxNodes = args[i + 1].toInteger() + assert maxNodes >= 0 + i += 1 + break + case '--min': + if (args.length <= i + 1) { + usageExit('Missing arguments after --min') + } + minNodes = args[i + 1].toInteger() + assert minNodes >= 0 + i += 1 + break + case '--machine': + if (args.length <= i + 1) { + usageExit('Missing arguments after --machine') + } + machineType = args[i + 1] + i += 1 + break + case '--current': + if (args.length <= i + 1) { + usageExit('Missing arguments after --current') + } + currentNodes = args[i + 1].toInteger() + i += 1 + break + default: + usageExit("Invalid argument: $arg") + } +} +if ((requiredRam > 0 || requiredCpu > 0) && (machineType == null)) { + usageExit("Machine type is required to calculate nodes from RAM or CPU required") +} +if ((requiredRam > 0 || requiredCpu > 0) && (requiredNodes != null)) { + usageExit("Cannot require Nodes and CPU or RAM") +} + +int machineRam = -1 +int machineCpu = -1 +if(machineType) { + switch (machineType) { + case 't2.small': + machineRam = 2 + machineCpu = 1 + break + case 't3.small': case 't3a.small': + machineRam = 2 + machineCpu = 2 + break + + case 'n2-highcpu-2': case 'n2d-highcpu-2': case 'e2-highcpu-2': + machineCpu = 2 + machineRam = 2 + break + case 'c2d-highcpu-2': case 't2.medium': case 't3.medium': case 't3a.medium': + machineRam = 4 + machineCpu = 2 + break + case 'n2-highcpu-4': case 'n2d-highcpu-4': case 'e2-highcpu-4': + machineRam = 4 + machineCpu = 4 + break + case 'n2-highmem-2': case 'n2d-highmem-2': case 'c2d-highmem-2': case 'e2-highmem-2': + machineRam = 16 + machineCpu = 2 + break + case 'n2-standard-2': case 'n2d-standard-2': case 'c2d-standard-2': case 'e2-standard-2': + case 'm5.large': case 'm5a.large': case 't2.large': case 't3a.large': case 't3.large': + machineRam = 8 + machineCpu = 2 + break + case 'c2d-highcpu-4': + machineRam = 8 + machineCpu = 4 + break + case 'n2-highcpu-8': case 'n2d-highcpu-8': case 'e2-highcpu-8': + machineRam = 8 + machineCpu = 8 + break + case 'n2-standard-4': case 'n2d-standard-4': case 'c2-standard-4': case 'c2d-standard-4': case 'e2-standard-4': + case 'm5.xlarge': case 'm5a.xlarge': case 't2.xlarge': case 't3a.xlarge': case 't3.xlarge': + machineRam = 16 + machineCpu = 4 + break + case 'c2d-highcpu-8': + machineRam = 16 + machineCpu = 8 + break + case 'n2-highcpu-16': case 'n2d-highcpu-16': case 'e2-highcpu-16': + machineRam = 16 + machineCpu = 16 + break + case 'n2-highmem-4': case 'n2d-highmem-4': case 'c2d-highmem-4': case 'e2-highmem-4': + machineRam = 32 + machineCpu = 4 + break + case 'n2-standard-8': case 'n2d-standard-8': case 'c2-standard-8': case 'c2d-standard-8': case 'e2-standard-8': + case 'm5.2xlarge': case 'm5a.2xlarge': case 't2.2xlarge': case 't3a.2xlarge': case 't3.2xlarge': + machineRam = 32 + machineCpu = 8 + break + case 'c2d-highcpu-16': + machineRam = 32 + machineCpu = 16 + break + case 'n2-highcpu-32': case 'n2d-highcpu-32': case 'e2-highcpu-32': + machineRam = 32 + machineCpu = 32 + break + case 'n2-highmem-8': case 'n2d-highmem-8': case 'c2d-highmem-8': case 'e2-highmem-8': + machineRam = 64 + machineCpu = 8 + break + case 'n2-standard-16': case 'n2d-standard-16': case 'c2-standard-16': case 'c2d-standard-16': case 'e2-standard-16': + case 'm5.4xlarge': case 'm5a.4xlarge': + machineRam = 64 + machineCpu = 16 + break + case 'c2d-highcpu-32': + machineRam = 64 + machineCpu = 32 + break + case 'n2-highcpu-64': case 'n2d-highcpu-64': + machineRam = 64 + machineCpu = 64 + break + case 'n2-highmem-16': case 'n2d-highmem-16': case 'c2d-highmem-16': case 'e2-highmem-16': + machineRam = 128 + machineCpu = 16 + break + default: + println "Unknown machine type $machineType" + System.exit(1) + } + if (verbose || outputFile != null) { + println "Machine Type:$machineType CPU=$machineCpu, RAM=$machineRam" + } +} + +int usedMemory = 0 +int usedCpu = 0 + +if (podFile != null && currentNodes > 0) { + Map sizes = calculatePodSizes(verbose, podFile) + if (verbose || outputFile != null) { + println "Pods sizes=$sizes" + } + if (sizes.memory != null) { + usedMemory = (int) Math.ceil(sizes.memory) + } + if (sizes.cpu != null) { + usedCpu = (int) Math.ceil(sizes.cpu) + } +} +int ramNodes = 0 +if (requiredRam != 0) { + int totalRam = currentNodes * machineRam + int availableRam = totalRam - usedMemory + if (requiredRam > availableRam) { + ramNodes = divideRoundUp(requiredRam - availableRam, machineRam) + } else if (requiredRam < 0 && Math.abs(requiredRam) < availableRam) { + ramNodes = divideRoundUp(availableRam - Math.abs(requiredRam), machineRam) * -1 + } + if (verbose || outputFile != null) { + println "RAM: total=$totalRam, used=$usedMemory, available=$availableRam, required=$requiredRam. Nodes=$ramNodes" + } +} +int cpuNodes = 0 +if (requiredCpu != 0) { + int totalCpu = currentNodes * machineCpu + int availableCpu = totalCpu - usedCpu + if (requiredCpu > availableCpu) { + cpuNodes = divideRoundUp(requiredCpu - availableCpu, machineRam) + } else if (requiredCpu < 0 && Math.abs(requiredCpu) < availableCpu) { + cpuNodes = divideRoundUp(availableCpu - Math.abs(requiredCpu), machineRam) * -1 + } + if (verbose || outputFile != null) { + println "CPU: total=$totalCpu, used=$usedCpu, available=$availableCpu, required=$requiredCpu. Nodes=$cpuNodes" + } +} +if (cpuNodes != 0 && ramNodes != 0) { + requiredNodes = Math.max(cpuNodes, ramNodes) +} else if (cpuNodes != 0) { + requiredNodes = cpuNodes +} else if (ramNodes != 0) { + requiredNodes = ramNodes +} +if (requiredNodes == null) { + requiredNodes = 0 +} +int targetNodes = currentNodes + requiredNodes +if (minNodes >= 0 && targetNodes < minNodes) { + if (verbose || outputFile != null) { + println "Min:$minNodes" + } + targetNodes = minNodes +} +if (maxNodes >= 0 && targetNodes > maxNodes) { + if (verbose || outputFile != null) { + println "Max:$maxNodes" + } + targetNodes = maxNodes + shrink = true +} +if (!shrink && targetNodes < currentNodes) { + targetNodes = currentNodes +} + +if (verbose || outputFile != null) { + println "Nodes: current=$currentNodes, required: $requiredNodes, target=$targetNodes, shrink=$shrink" +} +def result = [nodes: targetNodes, shrink: shrink] +String output = JsonOutput.toJson(result) +if (outputFile != null) { + def file = new File(outputFile) + file.text = output + if (verbose) { + println "Created: $outputFile" + } +} else { + println output +} diff --git a/scripts/calculate-pod-sizes.groovy b/scripts/calculate-pod-sizes.groovy new file mode 100644 index 00000000..60b79f19 --- /dev/null +++ b/scripts/calculate-pod-sizes.groovy @@ -0,0 +1,149 @@ +// Collect all pods. +// find container that are not completed and determine their size to calculate the total memory used by all the pods. + +import groovy.json.JsonOutput +import groovy.json.JsonSlurper + +import java.util.stream.Collectors + +double stringToGb(boolean verbose, String memory) { + double result = 0.5 + if (memory.endsWith('Mi')) { + result = Double.parseDouble(memory - 'Mi') / 1024.0 + } else if (memory.endsWith('Mb')) { + result = Double.parseDouble(memory - 'Mb') / 1000.0 + } else if (memory.endsWith('Gi')) { + result = Double.parseDouble(memory - 'Gi') + } else if (memory.endsWith('Gb')) { + result = Double.parseDouble(memory - 'Gb') + } else if (memory.endsWith('Ki')) { + result = Double.parseDouble(memory - 'Ki') / (1024.0 * 1024.0) + } else if (memory.endsWith('Kb')) { + result = Double.parseDouble(memory - 'Kb') / 1000000.0 + } + if (verbose) { + println "$memory=$result G" + } + return result +} + +double stringToCpu(boolean verbose, String cpu) { + double result = 0.1 + if (cpu.endsWith('m')) { + result = Double.parseDouble(cpu - 'm') / 1000.0 + } else { + result = Double.parseDouble(cpu) + } + if (verbose) { + println "$cpu=$result" + } + return result +} + +double memoryCalc(boolean verbose, String limit, String request) { + if (request != null) { + return stringToGb(verbose, request) + } + if (limit != null) { + return stringToGb(verbose, limit) + } + return 0.5 +} + +double cpuCalc(boolean verbose, String limit, String request) { + if (request != null) { + return stringToCpu(verbose, request) + } + if (limit != null) { + return stringToCpu(verbose, limit) + } + return 0.1 +} + +double calculateUsing(boolean verbose, Map matrix, Closure calc) { + return matrix.items.stream().filter { pod -> + if (verbose) { + println "Pod:$pod.metadata" + } + pod.status.containerStatuses.stream().anyMatch { containerStatus -> + if (verbose) { + println "\tcontainerStatus:$containerStatus" + } + boolean running = containerStatus.state?.running != null + if (verbose) { + println "Pod:${pod.metadata.name}:$running" + } + return running + } + }.map { pod -> + double m = pod.spec.containers.stream().map { container -> + double result = calc.call(pod, container) + if (verbose) { + println "Pod:${pod.metadata.name}:Container:${container.name}=$result" + } + return result + }.collect(Collectors.summingDouble(Double::doubleValue)) + if (verbose) { + println "Pod:${pod.metadata.name}=$m" + } + return m + }.collect(Collectors.summingDouble(Double::doubleValue)) +} + +Map calculatePodSizes(boolean verbose, String fileName) { + def file = new File(fileName) + if (!file.exists()) { + System.err.println "File not found: $file.absoluteFile" + System.exit(2) + } + JsonSlurper jsonSlurper = new JsonSlurper() + if (verbose) { + println "Loading: $file.absoluteFile" + } + def matrix = jsonSlurper.parse(file) + double memory = calculateUsing(verbose, matrix) { pod, container -> + double result = memoryCalc(verbose, container.resources.limits?.memory, container.resources.requests?.memory) + if (verbose) { + println "Pod:${pod.metadata.name}:Container:${container.name}=$result" + } + return result + } + double cpu = calculateUsing(verbose, matrix) { pod, container -> + double result = cpuCalc(verbose, container.resources.limits?.cpu, container.resources.requests?.cpu) + if (verbose) { + println "Pod:${pod.metadata.name}:Container:${container.name}=$result" + } + return result + } + return ['memory': memory, 'cpu': cpu] +} + +if (args.length == 0) { + println 'Usage: [--verbose] []' + System.exit(1) +} +boolean verbose = false +double totalMemory = 0.0 +double totalCpu = 0.0 +for (arg in args) { + if (arg == '--verbose') { + verbose = true + } else { + Map sizes = calculatePodSizes(verbose, arg) + if (verbose) { + println "File:$arg:sizes=$sizes" + } + if (sizes['memory'] != null) { + totalMemory += sizes['memory'] + } + if (sizes['cpu'] != null) { + totalCpu += sizes['cpu'] + } + } +} +long memory = (long) Math.ceil(totalMemory) +long cpu = (long) Math.ceil(totalCpu) +Map values = ['memory': memory, 'cpu': cpu] +def result = JsonOutput.toJson(values) +println "$result" + diff --git a/scripts/check-runners.sh b/scripts/check-runners.sh new file mode 100755 index 00000000..9a917b1d --- /dev/null +++ b/scripts/check-runners.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +SCDIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")") +SCDIR=$(realpath $SCDIR) +PARENT=$(realpath $SCDIR/..) +SCALING=$(jq '.scdf_pro_gh_runners.runner_scaling' $PARENT/config/defaults.json | sed 's/\"//g') +if [ "$SCALING" == "auto" ]; then + echo "Auto scaling:" + kubectl get horizontalrunnerautoscalers +fi +echo "" +echo "RunnerDeployments:" +DEPLOYMENTS=$(kubectl get rdeploy) +echo "$DEPLOYMENTS" +echo "" +echo "Runners:" +RUNNER_DEPLOYMENTS=$(echo "$DEPLOYMENTS" | grep -F "runner" | awk '{print $1}') +for deployment in $RUNNER_DEPLOYMENTS; do + kubectl get runners -l runner-deployment-name=$deployment --output=json | jq '.items | map(.status) | group_by(.phase,.ready) | map({ "deployment": '"\"$deployment\""', "count": length, "phase": .[0].phase})' +done diff --git a/scripts/create-cluster-tf-gke.sh b/scripts/create-cluster-tf-gke.sh new file mode 100755 index 00000000..9d861607 --- /dev/null +++ b/scripts/create-cluster-tf-gke.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +SCDIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")") +SCDIR=$(realpath $SCDIR) + +(return 0 2>/dev/null) && sourced=1 || sourced=0 +function check_env() { + eval ev='$'$1 + if [ "$ev" == "" ]; then + echo "$1 not defined" + if (( sourced != 0 )); then + return 1 + else + exit 1 + fi + fi +} + +set -e +check_env CLUSTER_NAME +check_env REGION +check_env GCP_CRED_JSON_FILE + +if [ "$K8S" == "" ]; then + export K8S="1.23" +fi +if [ "$NODES" == "" ]; then + export NODES=1 +fi + +# Machine types +# e2-standard-2 2 8 128 257 No 4 +# e2-standard-4 4 16 128 257 No 8 +# e2-standard-8 8 32 128 257 No 16 +# e2-standard-16 16 64 128 257 No 16 +# e2-standard-32 32 128 128 257 No 16 + +$SCDIR/terraform/tf-apply.sh $1 + +echo "Google Cloud console for cluster at https://console.cloud.google.com/kubernetes/clusters/details/$REGION/$CLUSTER_NAME/details?project=spring-cloud-dataflow-148214" + diff --git a/scripts/create-cluster-tmc.sh b/scripts/create-cluster-tmc.sh new file mode 100755 index 00000000..4ba915b3 --- /dev/null +++ b/scripts/create-cluster-tmc.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +(return 0 2>/dev/null) && sourced=1 || sourced=0 +if [ "$sourced" = "0" ]; then + echo "This script must be invoked using: source $0 $*" + exit 1 +fi +function check_env() { + eval ev='$'$1 + if [ "$ev" == "" ]; then + echo "$1 not defined" + if (( sourced != 0 )); then + return 1 + else + exit 1 + fi + fi +} +set -e +check_env CLUSTER_NAME +check_env K8S +check_env NODES + +sed -i 's/name-changeme/'"$CLUSTER_NAME"'/g' .github/tmc/dataflow-tmc-template-mod.yaml +sed -i 's/version-changeme/'"$K8S"'/g' .github/tmc/dataflow-tmc-template-mod.yaml +sed -i 's/nodes-changeme/'"$NODES"'/g' .github/tmc/dataflow-tmc-template-mod.yaml +tmc cluster create -f .github/tmc/dataflow-tmc-template-mod.yaml diff --git a/scripts/create-runners-cluster-gke.sh b/scripts/create-runners-cluster-gke.sh new file mode 100755 index 00000000..d414f897 --- /dev/null +++ b/scripts/create-runners-cluster-gke.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +SCDIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")") +SCDIR=$(realpath $SCDIR) + +export CLUSTER_NAME="stream-apps-gh-runners" +if [ "$REGION" == "" ]; then + export REGION="us-central1" +fi + +# MACHINE_TYPE="e2-standard-4" +export K8S="1.23" +export MACHINE_TYPE=$($SCDIR/determine-default.sh $CLUSTER_NAME "machine_type") +export DISK_SIZE=$($SCDIR/determine-default.sh $CLUSTER_NAME "disk_size") +export NODES=1 +export PODS_PER_NODE=30 + + +echo "Creating stream-apps-gh-runners" +$SCDIR/create-cluster-tf-gke.sh stream-apps-gh-runners diff --git a/scripts/create-runners-cluster-tmc.sh b/scripts/create-runners-cluster-tmc.sh new file mode 100755 index 00000000..4c79eb64 --- /dev/null +++ b/scripts/create-runners-cluster-tmc.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash +SCDIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")") +SCDIR=$(realpath $SCDIR) +PARENT=$(realpath $SCDIR/..) + +export CLUSTER_NAME="stream-apps-gh-runners" +echo "Creating stream-apps-gh-runners" +tmc cluster create -f $PARENT/.github/tmc/gh-runner-template.yaml diff --git a/scripts/decrease-runners-tmc.sh b/scripts/decrease-runners-tmc.sh new file mode 100755 index 00000000..99eae870 --- /dev/null +++ b/scripts/decrease-runners-tmc.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +SCDIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")") +SCDIR=$(realpath $SCDIR) + +function count_runners() { + kubectl get pods --namespace default | grep Running | grep -c "runners-ci" +} + +if [ "$1" = "" ]; then + echo "Expected number to decrease" + exit 1 +fi +mkdir -p $HOME/.kube +echo "Connecting to stream-apps-gh-runners" +tmc cluster auth kubeconfig get stream-apps-gh-runners > $HOME/.kube/runners-config +export KUBECONFIG=$HOME/.kube/runners-config +$SCDIR/decrease-runners.sh $* \ No newline at end of file diff --git a/scripts/decrease-runners.sh b/scripts/decrease-runners.sh new file mode 100755 index 00000000..35f0c5de --- /dev/null +++ b/scripts/decrease-runners.sh @@ -0,0 +1,93 @@ +#!/usr/bin/env bash +SCDIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")") +SCDIR=$(realpath $SCDIR) +PARENT=$(realpath $SCDIR/..) + +function max_replicas() { + kubectl get horizontalrunnerautoscalers --output=json | jq '.items | map(select(.spec.scaleTargetRef.name == "runners-ci")) | .[] | .spec.maxReplicas' +} +function min_replicas() { + kubectl get horizontalrunnerautoscalers --output=json | jq '.items | map(select(.spec.scaleTargetRef.name == "runners-ci")) | .[] | .spec.minReplicas' +} +function count_runners() { + kubectl get rdeploy runners-ci --output=json | jq '.spec.replicas' +} +function count_running() { + kubectl get rdeploy | grep -F "runners-ci" | awk '{print $4}' +} + +if [ "$1" = "" ]; then + echo "Expected number to decrease" + exit 1 +fi +echo "Checking stream-apps-gh-runners" +DEC=$1 +MIN_RUNNERS=$2 + +SCALING=$(jq '.scdf_pro_gh_runners.runner_scaling' $PARENT/config/defaults.json | sed 's/\"//g') +cp $SCDIR/k8s/runners-ci-${SCALING}-template.yaml runners-ci-change.yaml +if [ "$SCALING" == "auto" ]; then + MAX_RUNNERS=$(max_replicas) + MAX_RUNNERS=$((MAX_RUNNERS - DEC)) + CURRENT=$(min_replicas) + if [ "$MIN_RUNNERS" == "" ]; then + MIN_RUNNERS=$((CURRENT - DEC)) + fi + if ((MAX_RUNNERS < 1)); then + MAX_RUNNERS=1 + fi + if ((MIN_RUNNERS < 1)); then + MIN_RUNNERS=1 + fi +else + CURRENT=$(count_runners) + echo "There are now $CURRENT runners" + TARGET=$((CURRENT - DEC)) +fi + +if [ "$MIN_RUNNERS" != "" ]; then + if ((TARGET < MIN_RUNNERS)); then + TARGET=$MIN_RUNNERS + fi +fi +if ((TARGET < 1)); then + TARGET=1 +fi +OS=$(uname) +OS="${OS//./L&}" +ARCH=$(uname -i) +case $ARCH in +"x86_64") + ARCH=amd64 + ;; +*) + echo "Architecture:$ARCH may not be supported by summerwind/actions-runner" + ;; +esac +IMAGE_SHA=$(wget -q -O- https://hub.docker.com/v2/repositories/summerwind/actions-runner/tags | jq --arg os $OS --arg arch $ARCH '.results | map(select(.name="latest")) | .[0].images | map(select(.os==$os and .architecture==$arch)) | .[0].digest' | sed 's/\"//g') +if [ "$IMAGE_SHA" == "" ]; then + IMAGE_SHA="latest" +fi +sed -i 's/tag-placeholder/'"$IMAGE_SHA"'/g' runners-ci-change.yaml +if [ "$SCALING" == "auto" ]; then + MAX_RUNNERS=$((MAX_RUNNERS - DEC)) + echo "Runners: changing scaling min: $MIN_RUNNERS, max: $MAX_RUNNERS" + sed -i 's/max-replicas-placeholder/'"$MAX_RUNNERS"'/g' runners-ci-change.yaml + sed -i 's/min-replicas-placeholder/'"$MIN_RUNNERS"'/g' runners-ci-change.yaml +else + echo "Runners:$CURRENT decrease by $DEC to $TARGET" + sed -i 's/replicas-placeholder/'"$TARGET"'/g' runners-ci-change.yaml +fi +cmp --silent "$SCDIR/k8s/runners-ci-${SCALING}-template.yaml" runners-ci-change.yaml +RC=$? +if ((RC != 0)); then + kubectl apply -f runners-ci-change.yaml + echo "Runners: changed to $RUNNERS" + if [ "$SCALING" != "auto" ]; then + $SCDIR/wait-k8s.sh 1 --for=condition=ready --timeout=1m pod -l runner-deployment-name=runners-ci --all-namespaces=true + fi +else + echo "Runners: unchanged" +fi +rm -f runners-ci-change.yaml +$SCDIR/check-runners.sh diff --git a/scripts/delete-cluster-gke.sh b/scripts/delete-cluster-gke.sh new file mode 100755 index 00000000..174dfa77 --- /dev/null +++ b/scripts/delete-cluster-gke.sh @@ -0,0 +1,120 @@ +#!/usr/bin/env bash +SCDIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")") +SCDIR=$(realpath $SCDIR) + +(return 0 2>/dev/null) && sourced=1 || sourced=0 +function check_env() { + eval ev='$'$1 + if [ "$ev" == "" ]; then + echo "$1 not defined" + if ((sourced != 0)); then + return 1 + else + exit 1 + fi + fi +} + +set -e +if [ "$1" != "" ]; then + export CLUSTER_NAME="$1" +else + check_env CLUSTER_NAME "$CLUSTER_NAME" +fi +set +e +COUNT=$(gcloud container clusters list | grep -c -F "$CLUSTER_NAME") +if [ $COUNT -eq 0 ]; then + echo "Cluster $CLUSTER_NAME does not exist" +else + echo "Cluster $CLUSTER_NAME found" +fi +if ((COUNT > 1)); then + CLUSTER=$(gcloud container clusters list | grep -F "$CLUSTER_NAME" | awk '{print $1}') + echo "Multiple clusters matching $CLUSTER_NAME" + gcloud container clusters list | grep -F "$CLUSTER_NAME" + if [ "$CLUSTER_NAME" != "$CLUSTER" ]; then + echo "No exact match for $CLUSTER_NAME" + fi +fi +export RGN=$(gcloud container clusters list | grep -F "$CLUSTER_NAME" | awk '{print $2}') +if [ "$RGN" == "" ]; then + if [ "$REGION" == "" ]; then + REGION=$($SCDIR/determine-default.sh $CLUSTER_NAME "region") + if [ "$REGION" == "" ]; then + echo "Region not provided" + exit 0 + fi + echo "Region not provided using default:$REGION" + fi + echo "Search for other cluster related resources in $REGION" +else + REGION=$RGN +fi +echo "Deleting cluster and resources: $CLUSTER_NAME" +if ((COUNT > 0)); then + source $SCDIR/kubeconfig-gke.sh $CLUSTER_NAME + $SCDIR/delete-namespaces.sh --nowait +fi +COUNT=$(gcloud compute firewall-rules list --format="table(name,network)" | grep -c -F "$CLUSTER_NAME") +if [ $COUNT -gt 0 ]; then + FIREWALLS=$(gcloud compute firewall-rules list --format="table(name,network)" | grep -F "$CLUSTER_NAME" | awk '{print $1}') + echo "FIREWALLS=$FIREWALLS" + for firewall in $FIREWALLS; do + echo "Deleting firewall: $firewall" + gcloud compute firewall-rules delete "$firewall" --quiet + done +else + echo "No firewall rules for $CLUSTER_NAME" +fi +COUNT=$(gcloud container clusters list | grep -c -F "$CLUSTER_NAME") +if [ $COUNT -ne 0 ]; then + echo "Deleting cluster: $CLUSTER_NAME from $REGION" + gcloud container clusters delete "${CLUSTER_NAME}" --region "$REGION" --quiet +else + echo "No cluster named $CLUSTER_NAME" +fi +COUNT=$(gcloud compute routes list | grep -c -F "$CLUSTER_NAME") +if [ $COUNT -gt 0 ]; then + ROUTES=$(gcloud compute routes list | grep -F "$CLUSTER_NAME" | awk '{print $1}') + echo "ROUTES=$ROUTES" + for route in $ROUTES; do + echo "Deleting route: $route" + gcloud compute routes delete "$route" --quiet + done +else + echo "No routes for $CLUSTER_NAME" +fi +COUNT=$(gcloud compute networks subnets list --regions="$REGION" | grep -c -F "$CLUSTER_NAME") +if [ $COUNT -ne 0 ]; then + SUBNETS=$(gcloud compute networks subnets list --regions="$REGION" | grep -F "$CLUSTER_NAME" | awk '{print $1}') + echo "SUBNETS=$SUBNETS" + for subnet in $SUBNETS; do + echo "Deleting subnet: $subnet from $REGION" + gcloud compute networks subnets delete $subnet --region "$REGION" --quiet + done +else + echo "No network subnets for $CLUSTER_NAME" +fi +COUNT=$(gcloud compute networks list | grep -c -F "$CLUSTER_NAME") +if [ $COUNT -gt 0 ]; then + NETWORKS=$(gcloud compute networks list | grep -F "$CLUSTER_NAME" | awk '{print $1}') + echo "NETWORKS=$NETWORKS" + for network in $NETWORKS; do + echo "Deleting network: $network" + gcloud compute networks delete "$network" --quiet + done +else + echo "No networks for $CLUSTER_NAME" +fi +COUNT=$(gcloud compute instances list | grep -c -F "$CLUSTER_NAME") +if [ $COUNT -gt 0 ]; then + INSTANCES=$(gcloud compute instances list | grep -F "$CLUSTER_NAME" | awk '{print $1}') + echo "INSTANCES=$INSTANCES" + for instance in $INSTANCES; do + ZONE=$(gcloud compute instances list | grep -F "$instance" | awk '{print $2}') + echo "Deleting instance: $instance from $ZONE" + gcloud compute instances delete "$instance" --quiet --zone $ZONE + done +else + echo "No instances for $CLUSTER_NAME" +fi diff --git a/scripts/delete-cluster-tf-gke.sh b/scripts/delete-cluster-tf-gke.sh new file mode 100755 index 00000000..f0cd2bfc --- /dev/null +++ b/scripts/delete-cluster-tf-gke.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +SCDIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")") +SCDIR=$(realpath $SCDIR) + +(return 0 2>/dev/null) && sourced=1 || sourced=0 +function check_env() { + eval ev='$'$1 + if [ "$ev" == "" ]; then + echo "$1 not defined" + if (( sourced != 0 )); then + return 1 + else + exit 1 + fi + fi +} +if [ "$K8S" == "" ]; then + export K8S="1.23" +fi +check_env CLUSTER_NAME "$CLUSTER_NAME" +export RGN=$(gcloud container clusters list | grep -F "$CLUSTER_NAME" | awk '{print $2}') +if [ "$RGN" == "" ]; then + if [ "$REGION" == "" ]; then + echo "Region not provided" + exit 0 + fi + echo "Search for other cluster related resources in $REGION" +else + REGION=$RGN +fi +set +e +if [ -f "$SCDIR/terraform/${CLUSTER_NAME}.tfstate" ] && [ "$REGION" != "" ]; then + $SCDIR/terraform/tf-destroy.sh $1 +fi +rm -f "$SCDIR/terraform/${CLUSTER_NAME}.tfstate*" +$SCDIR/delete-cluster-gke.sh diff --git a/scripts/delete-cluster-tmc.sh b/scripts/delete-cluster-tmc.sh new file mode 100755 index 00000000..31c16d8d --- /dev/null +++ b/scripts/delete-cluster-tmc.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash +(return 0 2>/dev/null) && sourced=1 || sourced=0 +function check_env() { + eval ev='$'$1 + if [ "$ev" == "" ]; then + echo "$1 not defined" + exit 0 + fi +} +if [ "$1" != "" ]; then + export CLUSTER_NAME="$1" +fi +check_env CLUSTER_NAME +set +e +COUNT=$(tmc cluster list | grep -c -F "$CLUSTER_NAME") +if ((COUNT > 0)); then + echo "Deleting cluster: $CLUSTER_NAME" + set +e + tmc cluster delete "$CLUSTER_NAME" + COUNT=$(tmc cluster list | grep -c -F "$CLUSTER_NAME") + while ((COUNT > 0)); do + set +e + STATUS=$(tmc cluster get $CLUSTER_NAME --output json | jq '.status.phase') + RC=$? + if ((RC != 0)); then + echo "Cannot find cluster" + exit 0 + fi + echo "Waiting to delete $CLUSTER_NAME. Status=$STATUS" + sleep 30 + set +e + COUNT=$(tmc cluster list | grep -c -F "$CLUSTER_NAME") + done +fi diff --git a/scripts/delete-k8s-ns.sh b/scripts/delete-k8s-ns.sh new file mode 100755 index 00000000..6f829c12 --- /dev/null +++ b/scripts/delete-k8s-ns.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +WAIT= +CARVEL=false +NS=default +while [ "$1" != "" ]; do + case $1 in + "--nowait") + WAIT="--wait=false" + ;; + "--wait") + WAIT="--wait=true" + ;; + "--carvel") + CARVEL=true + ;; + *) + NS="$1" + ;; + esac + shift +done +set +e +if [ "$VERBOSE" != "" ]; then + kubectl get namespaces +fi +FOUND=$(kubectl get namespaces --output=json | jq --arg namespace $NS '.items | map(select(.metadata.name == $namespace)) | .[] | .metadata.name') +if [ "$VERBOSE" != "" ]; then + echo "FOUND=$FOUND" +fi +if [ "$FOUND" != "" ]; then + echo "Deleting all resources in Namespace: $NS" + if [ "$CARVEL" == "true" ]; then + kubectl delete packagerepositories --all $WAIT --namespace="$NS" + kubectl delete packageinstalls --all $WAIT --namespace="$NS" + kubectl delete apps --all $WAIT --namespace="$NS" + fi + kubectl delete deployments --all $WAIT --namespace="$NS" + kubectl delete statefulsets --all $WAIT --namespace="$NS" + kubectl delete svc --all $WAIT --namespace="$NS" + kubectl delete all --all $WAIT --namespace="$NS" + kubectl delete pods --all $WAIT --namespace="$NS" + kubectl delete secrets --all $WAIT --namespace="$NS" + kubectl delete pvc --all $WAIT --namespace="$NS" + if [ "$NS" != "default" ]; then + set +e + kubectl delete namespace "$NS" $WAIT --timeout=1m + RC=$? + set -e + if ((RC == 0)); then + echo "Deleted Namespace: $NS" + else + echo "Error $RC deleting Namespace: $NS" + fi + else + echo "Cleaned Namespace: $NS" + fi +else + echo "Namespace:$NS doesn't exist" +fi diff --git a/scripts/delete-namespaces.sh b/scripts/delete-namespaces.sh new file mode 100755 index 00000000..91abc18e --- /dev/null +++ b/scripts/delete-namespaces.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +SCDIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")") +SCDIR=$(realpath $SCDIR) + +(return 0 2>/dev/null) && sourced=1 || sourced=0 +NAMES_JSON=$(kubectl get namespaces --output=json | jq '.items | map({name:.metadata.name}) | map(select(.name | startswith("kube") | not)) | map(select(.name != "default"))') +NAMES=$(echo "$NAMES_JSON" | jq '.[] | .name' | sed 's/\"//g') +CARVEL=$(echo "$NAMES_JSON" | jq 'map(select(.name == "kapp-controller")) | .[] | .name') +if [ "$CARVEL" != "" ]; then + CARVEL=--carvel +fi +while [ "$1" != "" ]; do + case $1 in + "--nowait") + WAIT="--nowait" + ;; + "--wait") + WAIT="--wait" + ;; + *) + FILTER="$1" + ;; + esac + shift +done +while [ "$1" != "" ]; do + NAMES=$(echo "$NAMES_JSON" | jq --arg value "$1" 'map(select(.name | contains($value)) | .[] | .name' | sed 's/\"//g') + for name in $NAMES; do + $SCDIR/delete-k8s-ns.sh $CARVEL $name $WAIT + done +done +if [ "$FILTER" == "" ]; then + for name in $NAMES; do + $SCDIR/delete-k8s-ns.sh $CARVEL $name $WAIT + done + $SCDIR/delete-k8s-ns.sh $CARVEL default $WAIT +fi diff --git a/scripts/delete-runners-gke.sh b/scripts/delete-runners-gke.sh new file mode 100755 index 00000000..70e92bc6 --- /dev/null +++ b/scripts/delete-runners-gke.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +SCDIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")") +SCDIR=$(realpath $SCDIR) + +export CLUSTER_NAME="stream-apps-gh-runners" +set +e +echo "Deleting stream-apps-gh-runners" +$SCDIR/delete-k8s-ns.sh actions-runner-system --nowait +$SCDIR/delete-k8s-ns.sh cert-manager --nowait +rm -f $SCDIR/terraform/stream-apps-gh-runners/stream-apps-gh-runners.tfstate* +kubectl delete rolebinding rolebinding-default-privileged-sa-ns_actions-runner-system +kubectl delete clusterrolebinding actions-runner-privileged-cluster-role-binding +$SCDIR/delete-cluster-gke.sh +exit 0 + diff --git a/scripts/delete-runners-tmc.sh b/scripts/delete-runners-tmc.sh new file mode 100755 index 00000000..fcb5814c --- /dev/null +++ b/scripts/delete-runners-tmc.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +SCDIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")") +SCDIR=$(realpath $SCDIR) + +export CLUSTER_NAME="stream-apps-gh-runners" +set +e +source $SCDIR/kubeconfig-tmc.sh +echo "Deleting stream-apps-gh-runners" +$SCDIR/delete-k8s-ns.sh actions-runner-system +$SCDIR/delete-k8s-ns.sh cert-manager +kubectl delete rolebinding rolebinding-default-privileged-sa-ns_actions-runner-system +kubectl delete clusterrolebinding actions-runner-privileged-cluster-role-binding +$SCDIR/delete-cluster-tmc.sh +exit 0 diff --git a/scripts/deploy-gh-runners-gke.sh b/scripts/deploy-gh-runners-gke.sh new file mode 100755 index 00000000..db9985f6 --- /dev/null +++ b/scripts/deploy-gh-runners-gke.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +SCDIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")") +SCDIR=$(realpath $SCDIR) + +echo "Authenticating against stream-apps-gh-runners" +source $SCDIR/kubeconfig-gke.sh stream-apps-gh-runners +echo "Configuring actions-runner-controller" +export NS="actions-runner-system" +$SCDIR/ensure-ns.sh $NS +set +e +$SCDIR/add-roles.sh "system:aggregate-to-edit" "system:aggregate-to-admin" "system:aggregate-to-view" +$SCDIR/deploy-gh-runners.sh + diff --git a/scripts/deploy-gh-runners-tmc.sh b/scripts/deploy-gh-runners-tmc.sh new file mode 100755 index 00000000..7c559924 --- /dev/null +++ b/scripts/deploy-gh-runners-tmc.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +SCDIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")") +SCDIR=$(realpath $SCDIR) +kubectl create clusterrolebinding actions-runner-privileged-cluster-role-binding \ + --clusterrole=vmware-system-tmc-psp-privileged \ + --group=system:authenticated +$SCDIR/deploy-gh-runners.sh +kubectl create rolebinding rolebinding-default-privileged-sa-ns_actions-runner-system \ + --namespace actions-runner-system \ + --clusterrole=vmware-system-tmc-psp-privileged \ + --user=system:serviceaccount:actions-runner-system:default \ No newline at end of file diff --git a/scripts/deploy-gh-runners.sh b/scripts/deploy-gh-runners.sh new file mode 100755 index 00000000..af5886f3 --- /dev/null +++ b/scripts/deploy-gh-runners.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +SCDIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")") +SCDIR=$(realpath $SCDIR) +PARENT=$(realpath $SCDIR/..) + +set +e +function check_env() { + eval ev='$'$1 + if [ "$ev" == "" ]; then + echo "$1 not defined" + if (( sourced != 0 )); then + return 1 + else + exit 1 + fi + fi +} +if [ "$GH_ARC_PAT" == "" ]; then + check_env GH_ARC_APP_ID + check_env GH_ARC_INSTALLATION_ID + check_env GH_ARC_PRIVATE_KEY +fi +echo "Configuring actions-runner-controller" +set +e +echo "Create Certificate Manager" +NS=cert-manager +SVC=cert-manager +$SCDIR/ensure-ns.sh $NS +kubectl apply -f https://github.com/jetstack/cert-manager/releases/latest/download/cert-manager.yaml +set -e +$SCDIR/wait-deployment.sh $SVC $NS +echo "Certificate Manager installed" +echo "Configuring actions-runner-controller" +set +e +export NS=actions-runner-system +export SVC=controller-manager +$SCDIR/ensure-ns.sh $NS +kubectl apply -f $SCDIR/k8s/pod-priorities.yaml +kubectl apply -f $SCDIR/k8s/pod-priorities.yaml --namespace $NS +if [ "$GH_ARC_PAT" != "" ]; then + echo "Using GH_ARC_PAT as github_token" + kubectl create secret generic "$SVC-secret" \ + --namespace $NS \ + --from-literal=github_token="$GH_ARC_PAT" +else + echo "Using GitHub App $GH_ARC_APP_ID / $GH_ARC_INSTALLATION_ID" + kubectl create secret generic "$SVC-secret" \ + --namespace $NS \ + --from-literal=github_app_id="${GH_ARC_APP_ID}" \ + --from-literal=github_app_installation_id="${GH_ARC_INSTALLATION_ID}" \ + --from-literal=github_app_private_key="${GH_ARC_PRIVATE_KEY}" +fi + +kubectl create secret docker-registry scdf-metadata-default --namespace $NS --docker-server=registry-1.docker.io --docker-username=$DOCKER_HUB_USERNAME --docker-password=$DOCKER_HUB_PASSWORD +kubectl create secret docker-registry scdf-metadata-default --namespace default --docker-server=registry-1.docker.io --docker-username=$DOCKER_HUB_USERNAME --docker-password=$DOCKER_HUB_PASSWORD + +helm repo add actions-runner-controller https://actions-runner-controller.github.io/actions-runner-controller +helm install --namespace $NS -f $SCDIR/arc/values.yml --wait actions-runner-controller actions-runner-controller/actions-runner-controller +# kubectl create -f https://github.com/actions-runner-controller/actions-runner-controller/releases/download/v0.25.2/actions-runner-controller.yaml + +set -e +#$SCDIR/wait-deployment.sh $SVC $NS +echo "Creating runners" +SCALING=$(jq '.scdf_pro_gh_runners.runner_scaling' $PARENT/config/defaults.json | sed 's/\"//g') +kubectl apply -f "$SCDIR/k8s/runners-ci-${SCALING}.yaml" +kubectl apply -f "$SCDIR/k8s/runners-generic.yaml" +if [ "$SCALING" != "auto" ]; then + $SCDIR/wait-k8s.sh 1 --for=condition=ready --timeout=1m pod -l runner-deployment-name=runners-ci +fi +$SCDIR/check-runners.sh diff --git a/scripts/determine-default.sh b/scripts/determine-default.sh new file mode 100755 index 00000000..d5c1a06d --- /dev/null +++ b/scripts/determine-default.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +SCDIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")") +SCDIR=$(realpath $SCDIR) +PARENT=$(realpath $SCDIR/..) + +(return 0 2>/dev/null) && sourced=1 || sourced=0 + +function determine_provider() { + CHECK_NAME=$1 + CHECK_NAME=${CHECK_NAME//\-/_} + CHECK_VALUE=$2 + CHECK_VALUE=${CHECK_VALUE//\-/_} + QUERY=".${CHECK_NAME}.${CHECK_VALUE}" + VALUE=$(jq ''"$QUERY"'' $PARENT/config/defaults.json | sed 's/\"//g') + if [ "$VALUE" == "" ] || [ "$VALUE" == "null" ]; then + KEYS=$(jq 'to_entries[] | .key' $PARENT/config/defaults.json | sed 's/\"//g') + CHECK_NAME= + PREFIX=$1 + PREFIX=${PREFIX//\-/_} + for key in $KEYS; do + if [[ "$PREFIX" == *"$key"* ]]; then + CHECK_NAME=$key + fi + done + if [ "$CHECK_NAME" != "" ]; then + QUERY=".${CHECK_NAME}.${CHECK_VALUE}" + VALUE=$(jq ''"$QUERY"'' $PARENT/config/defaults.json | sed 's/\"//g') + fi + if [ "$VALUE" == "" ] || [ "$VALUE" == "null" ]; then + CHECK_NAME=default + QUERY=".${CHECK_NAME}.${CHECK_VALUE}" + VALUE=$(jq ''"$QUERY"'' $PARENT/config/defaults.json | sed 's/\"//g') + fi + fi + echo "$VALUE" +} + +if [ "$2" == "" ]; then + echo "Usage: $0 " + if ((sourced != 0)); then + return 1 + else + exit 1 + fi +fi +BLOCK="$1" +FIELD="$2" + +VALUE=$(determine_provider "$BLOCK" "$FIELD") +export VALUE +echo "$VALUE" +if [ "$VALUE" == "" ] || [ "$VALUE" == "null" ]; then + exit 1 +fi \ No newline at end of file diff --git a/scripts/determine-machine-type.sh b/scripts/determine-machine-type.sh new file mode 100644 index 00000000..52f54def --- /dev/null +++ b/scripts/determine-machine-type.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +function check_env() { + eval ev='$'$1 + if [ "$ev" == "" ]; then + echo "$1 not defined" + exit 1 + fi +} +check_env CLUSTER_NAME +check_env REGION +set +e +MACHINE_TYPE=$(gcloud container clusters describe $CLUSTER_NAME --region $REGION "--format=table(nodeConfig.machineType)" 2>/dev/null | grep -v "MACHINE_TYPE") +RC=$? +if ((RC > 0)); then + exit 0 +fi +echo "$MACHINE_TYPE" diff --git a/scripts/determine-provider.sh b/scripts/determine-provider.sh new file mode 100755 index 00000000..465249fb --- /dev/null +++ b/scripts/determine-provider.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +SCDIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")") +SCDIR=$(realpath $SCDIR) +PARENT=$(realpath $SCDIR/..) + +(return 0 2>/dev/null) && sourced=1 || sourced=0 +function check_env() { + eval ev='$'$1 + if [ "$ev" == "" ]; then + echo "$1 not defined" + if ((sourced != 0)); then + return 1 + else + exit 1 + fi + fi +} + +if [ "$1" != "" ]; then + CLUSTER_NAME=$1 +fi +check_env CLUSTER_NAME + +PROVIDER=$($SCDIR/determine-default.sh $CLUSTER_NAME "provider") +if [ ! -f "$SCDIR/use-${PROVIDER}.sh" ]; then + echo "Invalid provider: $PROVIDER. Use one of gke, tmc" + exit 1 +fi +export PROVIDER +echo "$PROVIDER" diff --git a/scripts/increase-runners.sh b/scripts/increase-runners.sh new file mode 100755 index 00000000..35662d7a --- /dev/null +++ b/scripts/increase-runners.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +SCDIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")") +SCDIR=$(realpath $SCDIR) +PARENT=$(realpath $SCDIR/..) + +function max_replicas() { + kubectl get horizontalrunnerautoscalers --output=json | jq '.items | map(select(.spec.scaleTargetRef.name == "runners-ci")) | .[] | .spec.maxReplicas' +} +function min_replicas() { + kubectl get horizontalrunnerautoscalers --output=json | jq '.items | map(select(.spec.scaleTargetRef.name == "runners-ci")) | .[] | .spec.minReplicas' +} +function count_runners() { + kubectl get rdeploy runners-ci --output=json | jq '.spec.replicas' +} +function count_running() { + kubectl get rdeploy | grep -F "runners-ci" | awk '{print $4}' +} + +if [ "$1" = "" ]; then + echo "Expected number to increase" + exit 1 +fi +echo "Checking stream-apps-gh-runners" +INC=$1 + +# shellcheck disable=SC2003 +MAX_RUNNERS=$2 +SCALING=$(jq '.scdf_pro_gh_runners.runner_scaling' $PARENT/config/defaults.json | sed 's/\"//g') +cp $SCDIR/k8s/runners-ci-${SCALING}-template.yaml runners-ci-change.yaml + +if [ "$SCALING" == "auto" ]; then + if [ "$MAX_RUNNERS" == "" ]; then + MAX_REPLICAS=$(max_replicas) + MAX_RUNNERS=$((MAX_REPLICAS + INC)) + else + if ((MAX_RUNNERS < MAX_REPLICAS)); then + MAX_RUNNERS=$MAX_REPLICAS + fi + fi + MIN_RUNNERS=$(min_replicas) + MIN_RUNNERS=$((MIN_RUNNERS + INC)) + if ((MIN_RUNNERS > MAX_RUNNERS)); then + MIN_RUNNERS=$MAX_RUNNERS + fi +else + CURRENT=$(count_runners) + echo "There are now $CURRENT runners" + TARGET=$((CURRENT + INC)) + if [ "$MAX_RUNNERS" != "" ]; then + MAX=$MAX_RUNNERS + if ((TARGET > MAX)); then + if ((CURRENT > MAX)); then + echo "There are $CURRENT which is more than $MAX" + exit 0 + fi + echo "There cannot be more than $MAX runners" + TARGET=$MAX + fi + fi +fi +OS=$(uname) +OS="${OS//./L&}" +ARCH=$(uname -i) +case $ARCH in +"x86_64") + ARCH=amd64 + ;; +*) + echo "Architecture:$ARCH may not be supported by summerwind/actions-runner" + ;; +esac +IMAGE_SHA=$(wget -q -O- https://hub.docker.com/v2/repositories/summerwind/actions-runner/tags | jq --arg os $OS --arg arch $ARCH '.results | map(select(.name="latest")) | .[0].images | map(select(.os==$os and .architecture==$arch)) | .[0].digest' | sed 's/\"//g') +if [ "$IMAGE_SHA" == "" ]; then + IMAGE_SHA="latest" +fi +sed -i 's/tag-placeholder/'"$IMAGE_SHA"'/g' runners-ci-change.yaml +if [ "$SCALING" == "auto" ]; then + echo "Runners: changing scaling min: $MIN_RUNNERS, max: $MAX_RUNNERS" + sed -i 's/max-replicas-placeholder/'"$MAX_RUNNERS"'/g' runners-ci-change.yaml + sed -i 's/min-replicas-placeholder/'"$MIN_RUNNERS"'/g' runners-ci-change.yaml +else + echo "Runners:$CURRENT increase by $INC to $RUNNERS" + sed -i 's/replicas-placeholder/'"$RUNNERS"'/g' runners-ci-change.yaml +fi + +cmp --silent "$SCDIR/k8s/runners-ci-${SCALING}-template.yaml" runners-ci-change.yaml +RC=$? +if ((RC != 0)); then + kubectl apply -f runners-ci-change.yaml + echo "Runners: applied change to $RUNNERS" + if [ "$SCALING" != "auto" ]; then + $SCDIR/wait-k8s.sh 1 --for=condition=ready --timeout=1m pod -l runner-deployment-name=runners-ci --all-namespaces=true + fi +else + echo "Runners: no changes required" +fi +rm -f runners-ci-change.yaml +$SCDIR/check-runners.sh diff --git a/scripts/is_there_only_one.sh b/scripts/is_there_only_one.sh new file mode 100755 index 00000000..a1761a54 --- /dev/null +++ b/scripts/is_there_only_one.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +if [ "$1" == "" ]; then + echo "Expect name of workflow to ignore" + exit 1 +fi +JOB="$1" +if [ "$2" != "" ]; then + RETRIES=$2 +else + RETRIES=1 +fi +JOBS=$(gh run list --json status,name --jq "map(select(.status == \"in_progress\" or .status == \"queued\")) | map(select(.name != \"$JOB\")) | length") +while ((JOBS != 0)); do + RETRIES=$((RETRIES - 1)) + if ((RETRIES <= 0)); then + echo "false" + exit 0 + fi + sleep 15 + JOBS=$(gh run list --json status,name --jq "map(select(.status == \"in_progress\" or .status == \"queued\")) | map(select(.name != \"$JOB\")) | length") +done +if ((JOBS == 0)); then + echo "true" +else + echo "false" +fi diff --git a/scripts/k8s/pod-priorities.yaml b/scripts/k8s/pod-priorities.yaml new file mode 100644 index 00000000..96c29e5e --- /dev/null +++ b/scripts/k8s/pod-priorities.yaml @@ -0,0 +1,7 @@ +apiVersion: scheduling.k8s.io/v1 +kind: PriorityClass +metadata: + name: high-priority +value: 1000000 +globalDefault: false +preemptionPolicy: Never diff --git a/scripts/k8s/runners-stream-ci-auto-template.yaml b/scripts/k8s/runners-stream-ci-auto-template.yaml new file mode 100644 index 00000000..dd9ed1f5 --- /dev/null +++ b/scripts/k8s/runners-stream-ci-auto-template.yaml @@ -0,0 +1,41 @@ +apiVersion: actions.summerwind.dev/v1alpha1 +kind: RunnerDeployment +metadata: + name: runners-stream-ci +spec: + template: + spec: + image: 'summerwind/actions-runner' + priorityClassName: high-priority + dockerdWithinRunnerContainer: true + imagePullPolicy: IfNotPresent + imagePullSecrets: + - name: stream-metadata-default + repository: spring-cloud/stream-applications + githubAPICredentialsFrom: + secretRef: + name: controller-manager-secret + resources: + requests: + cpu: "1000m" + memory: "1Gi" + labels: + - stream-ci +--- +apiVersion: actions.summerwind.dev/v1alpha1 +kind: HorizontalRunnerAutoscaler +metadata: + name: runners-stream-ci-scaling +spec: + githubAPICredentialsFrom: + secretRef: + name: controller-manager-secret + minReplicas: min-replicas-placeholder + maxReplicas: max-replicas-placeholder + scaleTargetRef: + kind: RunnerDeployment + name: runners-stream-ci + scaleUpTriggers: + - githubEvent: + workflowJob: {} + duration: "30m" diff --git a/scripts/k8s/runners-stream-ci-auto.yaml b/scripts/k8s/runners-stream-ci-auto.yaml new file mode 100644 index 00000000..2d403e77 --- /dev/null +++ b/scripts/k8s/runners-stream-ci-auto.yaml @@ -0,0 +1,41 @@ +apiVersion: actions.summerwind.dev/v1alpha1 +kind: RunnerDeployment +metadata: + name: runners-stream-ci +spec: + template: + spec: + priorityClassName: high-priority + image: 'summerwind/actions-runner:latest' + dockerdWithinRunnerContainer: true + imagePullPolicy: IfNotPresent + imagePullSecrets: + - name: stream-metadata-default + repository: spring-cloud/stream-applications + githubAPICredentialsFrom: + secretRef: + name: controller-manager-secret + resources: + requests: + cpu: "1000m" + memory: "1Gi" + labels: + - stream-ci +--- +apiVersion: actions.summerwind.dev/v1alpha1 +kind: HorizontalRunnerAutoscaler +metadata: + name: runners-stream-ci-scaling +spec: + githubAPICredentialsFrom: + secretRef: + name: controller-manager-secret + minReplicas: 2 + maxReplicas: 2 + scaleTargetRef: + kind: RunnerDeployment + name: runners-stream-ci + scaleUpTriggers: + - githubEvent: + workflowJob: {} + duration: "30m" diff --git a/scripts/k8s/runners-stream-ci-manual-template.yaml b/scripts/k8s/runners-stream-ci-manual-template.yaml new file mode 100644 index 00000000..c883ac89 --- /dev/null +++ b/scripts/k8s/runners-stream-ci-manual-template.yaml @@ -0,0 +1,24 @@ +apiVersion: actions.summerwind.dev/v1alpha1 +kind: RunnerDeployment +metadata: + name: runners-stream-ci +spec: + replicas: replicas-placeholder + template: + spec: + image: 'summerwind/actions-runner' + dockerdWithinRunnerContainer: true + priorityClassName: high-priority + imagePullPolicy: IfNotPresent + imagePullSecrets: + - name: stream-metadata-default + repository: spring-cloud/stream-applications + githubAPICredentialsFrom: + secretRef: + name: controller-manager-secret + resources: + requests: + cpu: "1000m" + memory: "1Gi" + labels: + - stream-ci diff --git a/scripts/k8s/runners-stream-ci-manual.yaml b/scripts/k8s/runners-stream-ci-manual.yaml new file mode 100644 index 00000000..5854ae0c --- /dev/null +++ b/scripts/k8s/runners-stream-ci-manual.yaml @@ -0,0 +1,24 @@ +apiVersion: actions.summerwind.dev/v1alpha1 +kind: RunnerDeployment +metadata: + name: runners-stream-ci +spec: + replicas: 2 + template: + spec: + image: 'summerwind/actions-runner:latest' + dockerdWithinRunnerContainer: true + priorityClassName: high-priority + imagePullPolicy: IfNotPresent + imagePullSecrets: + - name: stream-metadata-default + repository: spring-cloud/stream-applications + githubAPICredentialsFrom: + secretRef: + name: controller-manager-secret + resources: + requests: + cpu: "1000m" + memory: "1Gi" + labels: + - stream-ci diff --git a/scripts/kubeconfig-gke.sh b/scripts/kubeconfig-gke.sh new file mode 100755 index 00000000..88f501a2 --- /dev/null +++ b/scripts/kubeconfig-gke.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +(return 0 2>/dev/null) && sourced=1 || sourced=0 +if (( sourced == 0 )); then + echo "This script must be invoked using: source $0 $*" + exit 1 +fi +if [ "$1" != "" ]; then + export CLUSTER_NAME="$1" +fi +if [ "$CLUSTER_NAME" = "" ]; then + echo "CLUSTER_NAME not defined" + return 1 +fi +if [ "$REGION" == "" ]; then + if [ "$REGION" == "" ]; then + if [ "$2" != "" ]; then + export REGION="$2" + else + export REGION=$(gcloud container clusters list | grep -F "$CLUSTER_NAME" | awk '{print $2}') + if [ "$REGION" == "" ]; then + echo "Cannot find REGION from $CLUSTER_NAME" + return 1 + fi + fi + fi +fi +export USE_GKE_GCLOUD_AUTH_PLUGIN=True +mkdir -p $HOME/.kube +echo "Connecting to $CLUSTER_NAME" +set +e +MAX=45 +export KUBECONFIG= +for i in $(seq $MAX); do + gcloud container clusters get-credentials $CLUSTER_NAME --region $REGION > $HOME/.kube/config + RC=$? + if [ "$RC" = "0" ]; then + break; + fi + sleep 20 +done +if [ "$RC" != "0" ]; then + echo "Error connecting to $CLUSTER_NAME" + exit $RC +fi +export KUBECONFIG=$HOME/.kube/config +echo "Connected to $CLUSTER_NAME" diff --git a/scripts/kubeconfig-runners.sh b/scripts/kubeconfig-runners.sh new file mode 100755 index 00000000..d2cf61f8 --- /dev/null +++ b/scripts/kubeconfig-runners.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +SCDIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")") +SCDIR=$(realpath $SCDIR) + +(return 0 2>/dev/null) && sourced=1 || sourced=0 +if (( sourced == 0 )); then + echo "This script must be invoked using: source $0 $*" + exit 1 +fi +export CLUSTER_NAME=stream-apps-gh-runners +set +e +PROVIDER=$($SCDIR/determine-provider.sh "$CLUSTER_NAME") +source $SCDIR/kubeconfig-${PROVIDER}.sh $CLUSTER_NAME diff --git a/scripts/kubeconfig-tmc.sh b/scripts/kubeconfig-tmc.sh new file mode 100755 index 00000000..41631e1b --- /dev/null +++ b/scripts/kubeconfig-tmc.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +(return 0 2>/dev/null) && sourced=1 || sourced=0 +if [ "$sourced" = "0" ]; then + echo "This script must be invoked using: source $0 $*" + exit 1 +fi +if [ "$CLUSTER_NAME" = "" ]; then + echo "CLUSTER_NAME not defined" + return 1 +fi +mkdir -p $HOME/.kube +echo "Connecting to $CLUSTER_NAME" +set +e +MAX=45 +export KUBECONFIG= +for i in $(seq $MAX); do + tmc cluster auth kubeconfig get $CLUSTER_NAME > $HOME/.kube/config + RC=$? + if [ "$RC" = "0" ]; then + break; + fi + sleep 20 +done +if [ "$RC" != "0" ]; then + echo "Error connecting to $CLUSTER_NAME" + return $RC +fi +export KUBECONFIG=$HOME/.kube/config +echo "Connected to $CLUSTER_NAME" diff --git a/scripts/limit-runners.sh b/scripts/limit-runners.sh new file mode 100755 index 00000000..6dfd4a0e --- /dev/null +++ b/scripts/limit-runners.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +SCDIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")") +SCDIR=$(realpath $SCDIR) +PARENT=$(realpath $SCDIR/..) + +function max_replicas() { + kubectl get horizontalrunnerautoscalers --output=json | jq '.items | map(select(.spec.scaleTargetRef.name == "runners-ci")) | .[] | .spec.maxReplicas' +} + +function count_runners() { + kubectl get rdeploy | grep -F "runners-ci" | awk '{print $2}' +} +function count_running() { + kubectl get rdeploy | grep -F "runners-ci" | awk '{print $4}' +} +if [ "$1" = "" ]; then + echo "Expected number to runners" + exit 1 +fi +CURRENT=$(count_runners) +echo "Connecting to stream-apps-gh-runners. $CURRENT runners" +TARGET=$1 +MAX_RUNNERS=$2 +if [ "$MAX_RUNNERS" != "" ]; then + if ((MAX_RUNNERS < TARGET)); then + TARGET=$MAX_RUNNERS + echo "Runners. Target reduced to $TARGET" + fi +fi +if ((TARGET < 0)); then + TARGET=1 +fi +OS=$(uname) +OS="${OS//./L&}" +ARCH=$(uname -i) +case $ARCH in +"x86_64") + ARCH=amd64 + ;; +*) + echo "Architecture:$ARCH may not be supported by summerwind/actions-runner" + ;; +esac +IMAGE_SHA=$(wget -q -O- https://hub.docker.com/v2/repositories/summerwind/actions-runner/tags | jq --arg os $OS --arg arch $ARCH '.results | map(select(.name="latest")) | .[0].images | map(select(.os==$os and .architecture==$arch)) | .[0].digest' | sed 's/\"//g') +if [ "$IMAGE_SHA" == "" ]; then + IMAGE_SHA="latest" +fi +SCALING=$(jq '.scdf_pro_gh_runners.runner_scaling' $PARENT/config/defaults.json | sed 's/\"//g') +cp $SCDIR/k8s/runners-ci-${SCALING}-template.yaml runners-ci-change.yaml +sed -i 's/tag-placeholder/'"$IMAGE_SHA"'/g' runners-ci-change.yaml +if [ "$SCALING" == "auto" ]; then + if [ "$MAX_RUNNERS" == "" ]; then + MAX_RUNNERS=$(max_replicas) + fi + echo "Runners: Max runners to $MAX_RUNNERS" + echo "Runners: changing scaling min: $TARGET, max: $MAX_RUNNERS" + sed -i 's/max-replicas-placeholder/'"$MAX_RUNNERS"'/g' runners-ci-change.yaml + sed -i 's/min-replicas-placeholder/'"$TARGET"'/g' runners-ci-change.yaml +else + echo "Runners:$CURRENT change to $TARGET" + sed -i 's/replicas-placeholder/'"$TARGET"'/g' runners-ci-change.yaml +fi +cmp --silent "$SCDIR/k8s/runners-ci-${SCALING}-template.yaml" runners-ci-change.yaml +RC=$? +if ((RC != 0)); then + kubectl apply -f runners-ci-change.yaml + echo "Runners: applying changes" + if [ "$SCALING" != "auto" ]; then + $SCDIR/wait-k8s.sh 1 --for=condition=ready --timeout=1m pod -l runner-deployment-name=runners-ci --all-namespaces=true + fi +else + echo "Runners:no changes required" +fi +rm -f runners-ci-change.yaml +$SCDIR/check-runners.sh diff --git a/scripts/list-clusters-gke.sh b/scripts/list-clusters-gke.sh new file mode 100755 index 00000000..f231bd62 --- /dev/null +++ b/scripts/list-clusters-gke.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +if [ "$1" = "" ]; then + echo "cluster name portion to match must be provided" + exit 1 +fi +set +e +COUNT=$(gcloud container clusters list | grep -c "$1") +if [ "$COUNT" != "0" ]; then + echo "There are $COUNT $1 clusters" + export CLUSTERS=$(gcloud container clusters list | grep "$1" | awk '{print $1}' | tr "\n" " " | tr "\r" " ") + for cluster in $CLUSTERS; do + echo " $cluster" + done +fi diff --git a/scripts/list-clusters-tmc.sh b/scripts/list-clusters-tmc.sh new file mode 100755 index 00000000..f442cd5d --- /dev/null +++ b/scripts/list-clusters-tmc.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +if [ "$1" = "" ]; then + echo "cluster name portion to match must be provided" + exit 1 +fi +set +e +COUNT=$(tmc cluster list | grep -c "$1") +if [ "$COUNT" != "0" ]; then + echo "There are $COUNT $1 clusters remaining" + CLUSTERS=$(tmc cluster list | grep "$1" | awk '{print $1}' | tr "\n" " " | tr "\r" " ") + for cluster in $CLUSTERS; do + echo " $cluster" + done +fi diff --git a/scripts/list-last-workflows.sh b/scripts/list-last-workflows.sh new file mode 100755 index 00000000..f473e436 --- /dev/null +++ b/scripts/list-last-workflows.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +WORKFLOWS=("gke-full-ci.yml" "gke-ga-ci.yml" "gke-main-ci.yml" "gke-quick-ga-ci.yml" "gke-quick-main-ci.yml" "gke-smoke-ga-ci.yml" "gke-smoke-main-ci.yml") +echo "Listing last workflows" +echo "[" > last.json +ITEMS=0 +for workflow in ${WORKFLOWS[@]}; do + WF_JSON=$(gh run list -w "$workflow" --limit 1 --json conclusion,event,databaseId,name,status,updatedAt) + CONCLUSION=$(echo "$WF_JSON" | jq '.[0].conclusion' | sed 's/\"//g') + if [ "$CONCLUSION" != "null" ] && [ "$CONCLUSION" != "" ]; then + STATUS=$(echo "$WF_JSON" | jq '.[0].status' | sed 's/\"//g') + UPDATED=$(echo "$WF_JSON" | jq '.[0].updatedAt' | sed 's/\"//g') + EVENT=$(echo "$WF_JSON" | jq '.[0].event' | sed 's/\"//g') + NAME=$(echo "$WF_JSON" | jq '.[0].name' | sed 's/\"//g') + ID=$(echo "$WF_JSON" | jq '.[0].databaseId') + if ((ITEMS > 0)); then + echo "," >> last.json + fi + ITEMS=$((ITEMS+1)) + echo "{\"updated\":\"$UPDATED\", \"status\":\"$STATUS\", \"conclusion\":\"$CONCLUSION\", \"event\":\"$EVENT\", \"id\":\"$ID\", \"name\":\"$NAME\"}" >> last.json + fi +done +echo "]" >> last.json +jq -s '.[] | sort_by(.updated) | reverse' last.json > sort_last.json diff --git a/scripts/list-workflow-runs.sh b/scripts/list-workflow-runs.sh new file mode 100755 index 00000000..eb1deca0 --- /dev/null +++ b/scripts/list-workflow-runs.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +if [ "$1" == "" ]; then + echo "Provide one or more workflow names like ci.yml that does produce download artefacts" + exit 1 +fi +while [ "$1" != "" ]; do + echo "Listing workflow runs for $1" + gh run list -w "$1" + shift +done diff --git a/scripts/report-pod-changes.sh b/scripts/report-pod-changes.sh new file mode 100755 index 00000000..08e5084c --- /dev/null +++ b/scripts/report-pod-changes.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +if [ "$1" != "" ]; then + NS="$1" +fi +if [ "$NS" == "" ]; then + NS=default +fi +PODS=$(kubectl get pods --namespace $NS --output=json | jq '.items[] | .metadata.name' | sed 's/\"//g') +for pod in $PODS; do + POD_JSON_FILE="${pod}.json" + POD_JSON=$(kubectl get pod $pod --namespace $NS --output=json) + if [ -f $POD_JSON_FILE ]; then + mv $POD_JSON_FILE "${POD_JSON_FILE}.prev" + echo "$POD_JSON" >$POD_JSON_FILE + echo "Saved: $POD_JSON_FILE" + diff --ignore-all-space $POD_JSON_FILE "${POD_JSON_FILE}.prev" + else + echo "$POD_JSON" >$POD_JSON_FILE + echo "Saved: $POD_JSON_FILE" + fi + POD_FILE="${pod}.txt" + POD_DESC=$(kubectl describe pod $pod --namespace $NS) + if [ -f $POD_FILE ]; then + mv $POD_FILE "${POD_FILE}.prev" + echo "$POD_DESC" >$POD_FILE + echo "Saved: $POD_FILE" + diff --ignore-all-space $POD_FILE "${POD_FILE}.prev" + else + echo "$POD_DESC" >$POD_FILE + echo "Saved: $POD_FILE" + fi +done diff --git a/scripts/save_gcp_cred_json.sh b/scripts/save_gcp_cred_json.sh new file mode 100755 index 00000000..2d8d2105 --- /dev/null +++ b/scripts/save_gcp_cred_json.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +if [ "$1" == "" ]; then + echo "Input required" + exit 2 +fi +export GCP_CRED_JSON="$1" +if [[ "$GCP_CRED_JSON" == *"=" ]]; then + export GCP_CRED_JSON="$(echo $1 | base64 -d)" +fi +GCP_CRED_JSON_FILE="$HOME/.gcloud_cred.json" +echo "$GCP_CRED_JSON" > $GCP_CRED_JSON_FILE +CLIENT_EMAIL=$(jq '.client_email' $GCP_CRED_JSON_FILE | sed 's/\"//g') +if [ "$CLIENT_EMAIL" == "" ]; then + echo "Cannot parse credentials" + exit 1 +fi +echo "Using $CLIENT_EMAIL" +export GCP_CRED_JSON_FILE=$GCP_CRED_JSON_FILE diff --git a/scripts/scale-cluster-nodes.sh b/scripts/scale-cluster-nodes.sh new file mode 100755 index 00000000..737629b7 --- /dev/null +++ b/scripts/scale-cluster-nodes.sh @@ -0,0 +1,188 @@ +#!/usr/bin/env bash +SCDIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")") +SCDIR=$(realpath $SCDIR) +PARENT=$(realpath $SCDIR/..) +(return 0 2>/dev/null) && sourced=1 || sourced=0 +function check_env() { + eval ev='$'$1 + if [ "$ev" == "" ]; then + echo "$1 not defined" + if ((sourced != 0)); then + return 1 + else + exit 1 + fi + fi +} +function div_round_up() { + DIVIDEND=$1 + DIVISOR=$2 + QUOTIENT=$((DIVIDEND / DIVISOR)) + if ((QUOTIENT * DIVISOR < DIVIDEND)); then + echo "$((QUOTIENT + 1))" + else + echo "$QUOTIENT" + fi +} + +function set_nodecount_machine_type() { + if [ "$PROVIDER" == "gke" ]; then + REGION=$(gcloud container clusters list 2>/dev/null | grep -F "$CLUSTER_NAME" | awk '{print $2}') + if [ "$REGION" == "" ]; then + echo "Cannot find $CLUSTER_NAME" + exit 2 + fi + export REGION + NODE_JSON=$(gcloud container node-pools list --region "$REGION" --cluster "$CLUSTER_NAME" --format=json) + NODE_POOL=$(echo "$NODE_JSON" | jq '.[0].name' | sed 's/\"//g') + if [ "$NODE_POOL" == "" ] || [ "$NODE_POOL" == "null" ]; then + echo "Node pool not found in $NODE_JSON" + exit 2 + fi + NODE_COUNT=$(gcloud container node-pools describe $NODE_POOL --region "$REGION" --cluster "$CLUSTER_NAME" "--format=table(name,initialNodeCount)" | grep -F "$NODE_POOL") + NODE_POOLS=$(gcloud container node-pools list --region "$REGION" --cluster "$CLUSTER_NAME" "--format=table(name,locations)" | grep -F "$NODE_POOL" | sed 's/,\ /,/g') + ZONES=$(echo "$NODE_POOLS" | awk '{print $2}' | sed 's/,/\n/g' | grep -c -F "$REGION") + INITIAL_NODE_COUNT=$(echo "$NODE_COUNT" | awk '{print $2}') + if [ "$INITIAL_NODE_COUNT" == "" ]; then + INITIAL_NODE_COUNT=0 + fi + if ((ZONES > 0)); then + CURRENT_NODES=$((INITIAL_NODE_COUNT * ZONES)) + else + ZONES=1 + CURRENT_NODES=$INITIAL_NODE_COUNT + fi + if [ "$DETERMINE_TYPE" == "true" ]; then + MACHINE_TYPE=$(gcloud container clusters describe $CLUSTER_NAME --region $REGION "--format=table(nodeConfig.machineType)" | grep -v "MACHINE_TYPE") + fi + elif [ "$PROVIDER" == "tmc" ]; then + PRESENT=$(tmc cluster list | grep -c -F $CLUSTER_NAME) + if ((PRESENT == 0)); then + echo "Cluster not found $CLUSTER_NAME" + exit 2 + fi + NODE_POOL=$(tmc cluster nodepool list --cluster-name $CLUSTER_NAME | grep -F "$CLUSTER_NAME" | awk '{print $1}') + if [ "$NODE_POOL" == "" ]; then + echo "TMC cluster $CLUSTER_NAME nodepool not found" + exit 2 + fi + NODE_POOL_JSON=$(tmc cluster nodepool get $NODE_POOL --cluster-name $CLUSTER_NAME --output json) + INITIAL_NODE_COUNT=$(echo "$NODE_POOL_JSON" | jq '.spec.workerNodeCount' | sed 's/\"//g') + CURRENT_NODES=$INITIAL_NODE_COUNT + if [ "$DETERMINE_TYPE" == "true" ]; then + MACHINE_TYPE=$(echo "$NODE_POOL_JSON" | jq '.spec.tkgAws.instanceType' | sed 's/\"//g') + fi + else + echo "Unsupported provider: $PROVIDER" + fi + echo "Node pool $NODE_POOL, current node count=$CURRENT_NODES in $ZONES zones. Machine type: $MACHINE_TYPE" + export CURRENT_NODES + export MACHINE_TYPE + export NODE_POOL + export ZONES +} + +function usage() { + echo "Usage $0 [--ram ] | [--nodes ] [--cpu ] [--max fixed-nodes] [--shrink]" + echo " If nodes and fixed-nodes is the same the node count will be set to that value" + echo " If nodes is negative the number will be reduced by that many nodes" + echo " If --ram then the machine type will be used to determine the number of nodes" + echo " --cpu is optional and will be used to scale cluster to minimum required" + echo " Negative values will trigger a shrink or scale down of the cluster." + echo " Positive values that can be satisfied by current usage will only result in scale down when --shrink is provided." +} +if [ "$1" == "" ]; then + echo "Cluster name required" + exit 1 +fi +export CLUSTER_NAME=$1 +DETERMINE_TYPE=false +ZONES=1 +VERBOSE=false +ARGS= +while [ "$2" != "" ]; do + case $2 in + "--cpu" | "--ram") + DETERMINE_TYPE=true + ;; + "--verbose") + VERBOSE=true + ;; + esac + if [ "$ARGS" != "" ]; then + ARGS="$ARGS $2" + else + ARGS="$2" + fi + shift + export ARGS +done +if [ "$VERBOSE" == "true" ]; then + echo "Determine provide for :$CLUSTER_NAME" +fi +export PROVIDER=$($SCDIR/determine-provider.sh $CLUSTER_NAME) +if [ "$VERBOSE" == "true" ]; then + echo "Connect to $CLUSTER_NAME on $PROVIDER" +fi +source $SCDIR/use-${PROVIDER}.sh $CLUSTER_NAME +source $SCDIR/kubeconfig-${PROVIDER}.sh $CLUSTER_NAME + +set_nodecount_machine_type +if [ "$DETERMINE_TYPE" == "true" ]; then + kubectl get pods --all-namespaces=true -o json >pods.json + PODFILE=$(realpath pods.json) + export ARGS="--machine $MACHINE_TYPE --podfile $PODFILE $ARGS" +fi +export ARGS="--current $CURRENT_NODES $ARGS" +if [ "$VERBOSE" == "true" ]; then + echo "groovy $SCDIR/calculate-nodes.groovy --output nodes.json $ARGS" +fi +groovy $SCDIR/calculate-nodes.groovy --output nodes.json $ARGS +RC=$? +if ((RC != 0)); then + exit $RC +fi +NODES=$(jq '.nodes' nodes.json) +SHRINK=$(jq '.shrink' nodes.json) +echo "NODES=$NODES, ZONES=$ZONES, SHRINK=$SHRINK" +TARGET=$(div_round_up $NODES $ZONES) +if ((TARGET < 0)); then + TARGET=$($SCDIR/determine-default.sh $CLUSTER_NAME "scale_down") +fi +if ((TARGET < INITIAL_NODE_COUNT)); then + if [ "$SHRINK" == "true" ]; then + echo "Scaling down $CLUSTER_NAME on $REGION and $ZONES zones from $INITIAL_NODE_COUNT to $TARGET per zone" + else + echo "Not scaling down. Leaving $CLUSTER_NAME on $INITIAL_NODE_COUNT nodes per zone" + TARGET= + fi +elif ((TARGET > INITIAL_NODE_COUNT)); then + echo "Scaling $CLUSTER_NAME on $REGION and $ZONES zones from $INITIAL_NODE_COUNT to $TARGET per zone" +else + echo "Leaving $CLUSTER_NAME on $INITIAL_NODE_COUNT nodes per zone" + TARGET= +fi +if [ "$TARGET" != "" ]; then + if [ "$PROVIDER" == "gke" ]; then + if [ "$DRY_RUN" == "true" ]; then + echo "gcloud container clusters resize "$CLUSTER_NAME" --region "$REGION" "--num-nodes=$TARGET" --node-pool=$NODE_POOL --quiet" + else + gcloud container clusters resize "$CLUSTER_NAME" --region "$REGION" "--num-nodes=$TARGET" --node-pool=$NODE_POOL --quiet + fi + elif [ "$PROVIDER" == "tmc" ]; then + if [ "$DRY_RUN" == "true" ]; then + echo "tmc cluster nodepool update $NODE_POOL --cluster-name $CLUSTER_NAME --worker-node-count $TARGET" + else + tmc cluster nodepool update $NODE_POOL --cluster-name $CLUSTER_NAME --worker-node-count $TARGET + STATUS=$(tmc cluster nodepool get $NODE_POOL --cluster-name $CLUSTER_NAME --output json | jq '.status.phase' | sed 's/\"//g') + while [ "$STATUS" != "READY" ]; do + echo "TMC Node pool $NODE_POOL for $CLUSTER_NAME status:$STATUS" + sleep 10 + STATUS=$(tmc cluster nodepool get $NODE_POOL --cluster-name $CLUSTER_NAME --output json | jq '.status.phase' | sed 's/\"//g') + done + echo "TMC Node pool $NODE_POOL for $CLUSTER_NAME status:$STATUS" + COUNT=$(tmc cluster nodepool get $NODE_POOL --cluster-name $CLUSTER_NAME --output json | jq '.spec.workerNodeCount' | sed 's/\"//g') + echo "TMC cluster $CLUSTER_NAME has $COUNT nodes" + fi + fi +fi diff --git a/scripts/scale-cluster-pods.sh b/scripts/scale-cluster-pods.sh new file mode 100755 index 00000000..203f075b --- /dev/null +++ b/scripts/scale-cluster-pods.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +SCDIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")") +SCDIR=$(realpath $SCDIR) +PARENT=$(realpath $SCDIR/..) + +(return 0 2>/dev/null) && sourced=1 || sourced=0 + +function check_env() { + eval ev='$'$1 + if [ "$ev" == "" ]; then + echo "$1 not defined" + if (( sourced != 0 )); then + return 1 + else + exit 1 + fi + fi +} + +if [ "$1" == "" ]; then + echo "Cluster name required" + exit 1 +fi +CLUSTER_NAME=$1 +if [ "$2" == "" ]; then + echo "POD count required" + exit 1 +fi +PODS=$2 +RAM_PER_POD=$($SCDIR/determine-default.sh $CLUSTER_NAME "ram-per-pod") +CPU_PER_POD=$($SCDIR/determine-default.sh $CLUSTER_NAME "cpu-per-pod") +PODS_PER_JOB=$($SCDIR/determine-default.sh $CLUSTER_NAME "pods_per_job") +TOTAL_RAM=$((RAM_PER_POD * PODS * PODS_PER_JOB)) +TOTAL_CPU=$((CPU_PER_POD * PODS * PODS_PER_JOB)) +echo "Scaling $CLUSTER_NAME with $PODS pods at ${RAM_PER_POD}Gi and $CPU_PER_POD vCPUs per pod using $PODS_PER_JOB per job. Total RAM:$TOTAL_RAM and CPU:$TOTAL_CPU" +ARGS= +while [ "$3" != "" ]; do + if [ "$ARGS" != "" ]; then + ARGS="$ARGS $3" + else + ARGS="$3" + fi + shift +done +$SCDIR/scale-cluster-nodes.sh $CLUSTER_NAME --ram $TOTAL_RAM --cpu $TOTAL_CPU $ARGS diff --git a/scripts/terraform/stream-apps-gh-runners/kube-admin.yaml b/scripts/terraform/stream-apps-gh-runners/kube-admin.yaml new file mode 100644 index 00000000..fb6f759d --- /dev/null +++ b/scripts/terraform/stream-apps-gh-runners/kube-admin.yaml @@ -0,0 +1,19 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: admin-user + namespace: kube-system +--- +# Create ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: admin-user +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: cluster-admin +subjects: + - kind: ServiceAccount + name: admin-user + namespace: kube-system \ No newline at end of file diff --git a/scripts/terraform/stream-apps-gh-runners/kubeconfig-template.yaml b/scripts/terraform/stream-apps-gh-runners/kubeconfig-template.yaml new file mode 100644 index 00000000..ee6f5f29 --- /dev/null +++ b/scripts/terraform/stream-apps-gh-runners/kubeconfig-template.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Config +preferences: + colors: true +current-context: ${cluster_name} +contexts: + - context: + cluster: ${cluster_name} + user: ${cluster_name} + name: ${cluster_name} +clusters: + - cluster: + server: https://${endpoint} + certificate-authority-data: ${cluster_ca} + name: ${cluster_name} +users: + - name: ${cluster_name} + user: + client-certificate-data: ${client_cert} + client-key-data: ${client_cert_key} \ No newline at end of file diff --git a/scripts/terraform/stream-apps-gh-runners/main.tf b/scripts/terraform/stream-apps-gh-runners/main.tf new file mode 100644 index 00000000..0cc10107 --- /dev/null +++ b/scripts/terraform/stream-apps-gh-runners/main.tf @@ -0,0 +1,172 @@ +variable "cluster_name" { + default = "scdf-empty" +} +variable "region" { + default = "us-central1" +} + +variable "project" { + default = "spring-cloud-dataflow-148214" +} +variable "disk_size" { + default = 10 +} + +variable "node_count" { + default = 1 +} + +variable "max_cluster_cpu" { + default = 96 +} +variable "max_cluster_memory" { + default = 192 +} + +variable "machine_type" { + default = "e2-standard-4" +} +variable "kubernetes_version" { + default = "1.23" +} +variable "gcp_creds_json_file" { +} +variable "node_port" { + default = "30000" +} +variable "service_account" { + default = "scdf-gke-admin@spring-cloud-dataflow-148214.iam.gserviceaccount.com" +} +variable "port_name" { + default = "http" +} +variable "pods_per_node" { + default = "50" +} +provider "google" { + project = var.project + region = var.region + credentials = var.gcp_creds_json_file +} + +resource "google_compute_network" "vpc" { + auto_create_subnetworks = false + delete_default_routes_on_create = false + description = "Compute Network for GKE nodes" + name = "${var.cluster_name}-vpc" +} + +resource "google_compute_subnetwork" "scdf" { + name = "${var.cluster_name}-subnet" + ip_cidr_range = "10.255.0.0/20" + region = var.region + network = google_compute_network.vpc.id + secondary_ip_range { + range_name = local.cluster_secondary_range_name + ip_cidr_range = "10.0.0.0/20" + } + + secondary_ip_range { + range_name = local.services_secondary_range_name + ip_cidr_range = "10.64.0.0/20" + } +} + +resource "google_container_node_pool" "nodes" { + name = "${var.cluster_name}-node-pool" + cluster = google_container_cluster.primary.id + node_count = var.node_count + node_config { + machine_type = var.machine_type + service_account = var.service_account + oauth_scopes = [ + "https://www.googleapis.com/auth/compute", + "https://www.googleapis.com/auth/devstorage.read_only", + "https://www.googleapis.com/auth/logging.write", + "https://www.googleapis.com/auth/trace.append", + "https://www.googleapis.com/auth/monitoring", + ] + + labels = { + env = var.cluster_name + } + disk_size_gb = var.disk_size + disk_type = "pd-ssd" + image_type = "UBUNTU_CONTAINERD" + tags = ["scdf-node", var.cluster_name, "${var.cluster_name}-node"] + } + management { + auto_upgrade = false + auto_repair = true + } + # autoscaling { + # min_node_count = 1 + # max_node_count = var.node_count + # } + timeouts { + create = "20m" + update = "10m" + } +} + +resource "google_container_cluster" "primary" { + name = var.cluster_name + location = var.region + network = google_compute_network.vpc.name + subnetwork = google_compute_subnetwork.scdf.name + node_locations = ["${var.region}-c"] # "${var.region}-a", "${var.region}-b", + initial_node_count = 1 + remove_default_node_pool = true + default_max_pods_per_node = var.pods_per_node + release_channel { + channel = "UNSPECIFIED" + } + addons_config { + http_load_balancing { + disabled = false + } + horizontal_pod_autoscaling { + disabled = false + } + } + ip_allocation_policy { + } +} + +resource "google_compute_firewall" "scdf-fw" { + name = "${var.cluster_name}-firewall" + network = google_compute_network.vpc.name + + allow { + protocol = "tcp" + ports = ["80", "443", "22", "6080", "10250", "9443", "30000-32767"] + } + + target_tags = ["${var.cluster_name}-node"] + source_ranges = ["0.0.0.0/0"] +} + + +data "google_container_cluster" "information" { + name = google_container_cluster.primary.name + location = google_container_cluster.primary.location + + depends_on = [ + google_compute_network.vpc + ] +} + +provider "kubernetes" { + host = google_container_cluster.primary.endpoint + token = data.google_client_config.primary.access_token + cluster_ca_certificate = base64decode(google_container_cluster.primary.master_auth.0.cluster_ca_certificate) + config_path = "~/.kube/config" + client_key = base64decode(google_container_cluster.primary.master_auth.0.client_key) +} + +locals { + cluster_secondary_range_name = "${var.cluster_name}-cluster-secondary-range" + services_secondary_range_name = "${var.cluster_name}-services-secondary-range" +} + +data "google_client_config" "primary" {} diff --git a/scripts/terraform/stream-apps-gh-runners/output.tf b/scripts/terraform/stream-apps-gh-runners/output.tf new file mode 100644 index 00000000..8d1d319e --- /dev/null +++ b/scripts/terraform/stream-apps-gh-runners/output.tf @@ -0,0 +1,19 @@ +output "region" { + value = var.region + description = "GCloud Region" +} + +output "project_id" { + value = var.project + description = "GCloud Project ID" +} + +output "kubernetes_cluster_name" { + value = google_container_cluster.primary.name + description = "GKE Cluster Name" +} + +output "kubernetes_cluster_host" { + value = google_container_cluster.primary.endpoint + description = "GKE Cluster Host" +} \ No newline at end of file diff --git a/scripts/terraform/stream-apps-gh-runners/terraform.tfvars b/scripts/terraform/stream-apps-gh-runners/terraform.tfvars new file mode 100644 index 00000000..b6420ba4 --- /dev/null +++ b/scripts/terraform/stream-apps-gh-runners/terraform.tfvars @@ -0,0 +1,10 @@ +cluster_name="stream-apps-gh-runners" +region="us-central1" +gcp_creds_json_file="/home/pcorneil/.gcloud/gcp_creds.json" +node_count=1 +kubernetes_version="1.23.8-gke.400" +machine_type="c2d-standard-2" +disk_size=100 +max_cluster_memory=192 +max_cluster_cpu=24 + diff --git a/scripts/terraform/tf-apply.sh b/scripts/terraform/tf-apply.sh new file mode 100755 index 00000000..99490c8c --- /dev/null +++ b/scripts/terraform/tf-apply.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +SCDIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")") +SCDIR=$(realpath $SCDIR) +PROJECT_DIR=$(realpath $SCDIR/../..) + +if [ "$1" == "" ]; then + echo "Require folder for terraform files" + exit 1 +fi +(return 0 2>/dev/null) && sourced=1 || sourced=0 +function check_env() { + eval ev='$'$1 + if [ "$ev" == "" ]; then + echo "$1 not defined" + if ((sourced != 0)); then + return 1 + else + exit 1 + fi + fi +} + +check_env CLUSTER_NAME "$CLUSTER_NAME" +check_env REGION "$REGION" +check_env GCP_CRED_JSON_FILE "$GCP_CRED_JSON_FILE" + +K8S="${K8S//\-/.}" +VERSIONS_JSON=$(cat $PROJECT_DIR/config/k8s-versions.json) +if [ "$K8S" == "" ]; then + K8S=$(echo "$VERSIONS_JSON" | jq '.default_version' | sed 's/\"//g') + echo "Choosing default K8S=$K8S" +fi +K8S_VERSION=$(echo "$VERSIONS_JSON" | jq ".versions | map(select(.k8s == \"$K8S\")) | .[].k8s_version" | sed 's/\"//g') +if [ "$K8S_VERSION" == "" ]; then + echo "Unsupported version: $K8S" + exit 1 +fi +echo "K8S=$K8S => $K8S_VERSION" + +TF_DIR="$SCDIR/$1/$K8S" +if [ ! -d "$TF_DIR" ]; then + TF_DIR="$SCDIR/$1" +fi +if [ ! -d "$TF_DIR" ]; then + echo "Directory not found $TF_DIR" + exit 2 +fi + +pushd $TF_DIR >/dev/null +echo "cluster_name=\"$CLUSTER_NAME\"" >terraform.tfvars +echo "region=\"$REGION\"" >>terraform.tfvars +echo "gcp_creds_json_file=\"$GCP_CRED_JSON_FILE\"" >>terraform.tfvars + +if [ "$NODES" != "" ]; then + echo "node_count=$NODES" >>terraform.tfvars +fi + +echo "kubernetes_version=\"$K8S_VERSION\"" >>terraform.tfvars +if [ "$MACHINE_TYPE" != "" ]; then + echo "machine_type=\"$MACHINE_TYPE\"" >>terraform.tfvars +fi +if [ "$DISK_SIZE" != "" ]; then + echo "disk_size=$DISK_SIZE" >>terraform.tfvars +fi +if [ "$MAX_MEMORY" != "" ]; then + echo "max_cluster_memory=$MAX_MEMORY" >>terraform.tfvars +fi +if [ "$MAX_CPU" != "" ]; then + echo "max_cluster_cpu=$MAX_CPU" >>terraform.tfvars +fi +if [ "$PODS_PER_NODE" != "" ]; then + echo "pods_per_node=$PODS_PER_NODE" >>terraform.tfvars +fi +echo "" >>terraform.tfvars +if [ ! -f ".terraform.lock.hcl" ] || [ ! -d ".terraform" ]; then + terraform init +fi +echo "Terraform parameters:" +cat ./terraform.tfvars +echo "Validating Terraform for $CLUSTER_NAME" +set -e +terraform validate -no-color +echo "Applying Terraform for $CLUSTER_NAME" +terraform apply -no-color -auto-approve -refresh=true -input=false -state="${CLUSTER_NAME}.tfstate" +popd >/dev/null diff --git a/scripts/terraform/tf-destroy.sh b/scripts/terraform/tf-destroy.sh new file mode 100755 index 00000000..dfcced65 --- /dev/null +++ b/scripts/terraform/tf-destroy.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +SCDIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")") +SCDIR=$(realpath $SCDIR) + +if [ "$1" == "" ]; then + echo "Require folder for terraform files" + exit 1 +fi +(return 0 2>/dev/null) && sourced=1 || sourced=0 +function check_env() { + eval ev='$'$1 + if [ "$ev" == "" ]; then + echo "$1 not defined" + if (( sourced != 0 )); then + return 1 + else + exit 1 + fi + fi +} +K8S="${K8S//\-/.}" +check_env CLUSTER_NAME "$CLUSTER_NAME" +check_env REGION "$REGION" +check_env GCP_CRED_JSON_FILE "$GCP_CRED_JSON_FILE" +TF_DIR="$SCDIR/$1/$K8S" +if [ ! -d "$TF_DIR" ]; then + TF_DIR="$SCDIR/$1" +fi +if [ ! -d "$TF_DIR" ]; then + echo "Directory not found $TF_DIR" +fi + +pushd $TF_DIR > /dev/null + echo "cluster_name=\"$CLUSTER_NAME\"" > terraform.tfvars + echo "region=\"$REGION\"" >> terraform.tfvars + echo "gcp_creds_json_file=\"$GCP_CRED_JSON_FILE\"" >> terraform.tfvars + echo "" >> terraform.tfvars + if [ ! -f ".terraform.lock.hcl" ] || [ ! -d ".terraform" ]; then + terraform init + fi + echo "Terraform parameters:$(cat ./terraform.tfvars)" + echo "Destroying $CLUSTER_NAME" + terraform destroy -auto-approve -refresh=true -no-color -input=false -state="${CLUSTER_NAME}.tfstate" +popd > /dev/null diff --git a/scripts/there_can_be_only_one.sh b/scripts/there_can_be_only_one.sh new file mode 100755 index 00000000..055559a0 --- /dev/null +++ b/scripts/there_can_be_only_one.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +if [ "$1" == "" ]; then + echo "Expect name of workflow to ignore" +fi +JOB="$1" +if [ "$2" != "" ]; then + RETRIES=$2 +else + RETRIES=10 +fi +JOBS=$(gh run list --json status,name --jq "map(select(.status == \"in_progress\" or .status == \"queued\")) | map(select(.name != \"$JOB\")) | length") +while ((JOBS != 0)); do + RETRIES=$((RETRIES - 1)) + if ((RETRIES <= 0)); then + echo "THERE CAN BE ONLY ONE!" + exit 1 + fi + echo "Jobs In progress or queued:" + gh run list --json status,name --jq "map(select(.status == \"in_progress\" or .status == \"queued\")) | map(select(.name != \"$JOB\")) | .[] | .name" + sleep 10 + JOBS=$(gh run list --json status,name --jq "map(select(.status == \"in_progress\" or .status == \"queued\")) | map(select(.name != \"$JOB\")) | length") +done +echo "There is only one." diff --git a/scripts/update-cluster-tf-gke.sh b/scripts/update-cluster-tf-gke.sh new file mode 100755 index 00000000..e53226c0 --- /dev/null +++ b/scripts/update-cluster-tf-gke.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +SCDIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")") +SCDIR=$(realpath $SCDIR) + +(return 0 2>/dev/null) && sourced=1 || sourced=0 +function check_env() { + eval ev='$'$1 + if [ "$ev" == "" ]; then + echo "$1 not defined" + if (( sourced != 0 )); then + return 1 + else + exit 1 + fi + fi +} +set -e +check_env CLUSTER_NAME "$CLUSTER_NAME" +export REGION=$(gcloud container clusters list | grep -F "$CLUSTER_NAME" | awk '{print $2}') +if [ "$REGION" == "" ]; then + echo "Cannot find $CLUSTER_NAME" + exit 2 +fi +check_env GCP_CRED_JSON_FILE "$GCP_CRED_JSON_FILE" + +# Machine types +# e2-standard-2 2 8 128 257 No 4 +# e2-standard-4 4 16 128 257 No 8 +# e2-standard-8 8 32 128 257 No 16 +# e2-standard-16 16 64 128 257 No 16 +# e2-standard-32 32 128 128 257 No 16 + +$SCDIR/terraform/tf-apply.sh $1 diff --git a/scripts/update-runner-values.sh b/scripts/update-runner-values.sh new file mode 100755 index 00000000..ae722809 --- /dev/null +++ b/scripts/update-runner-values.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash +SCDIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")") +SCDIR=$(realpath $SCDIR) +PARENT=$(realpath $SCDIR/..) + +helm upgrade actions-runner-controller -f $SCDIR/arc/values.yml --namespace actions-runner-system actions-runner-controller/actions-runner-controller \ No newline at end of file diff --git a/scripts/use-gke.sh b/scripts/use-gke.sh new file mode 100755 index 00000000..8b38d5df --- /dev/null +++ b/scripts/use-gke.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +SCDIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")") +SCDIR=$(realpath $SCDIR) + +(return 0 2>/dev/null) && sourced=1 || sourced=0 +if (( sourced == 0 )); then + echo "This script must be invoked using: source $0 $*" + exit 1 +fi +function usage() { + echo "Usage is $0 " + if (( sourced != 0 )); then + exit 1 + else + return 1 + fi +} + +if [ "$1" == "" ]; then + usage +fi +export CLUSTER_NAME="$1" +echo "Determining region for $CLUSTER_NAME" +export REGION=$(gcloud container clusters list | grep -F "$CLUSTER_NAME" | awk '{print $2}') +if [ "$REGION" == "" ]; then + echo "Cannot find $CLUSTER_NAME" + if (( sourced != 0 )); then + exit 1 + else + return 1 + fi +fi +echo "Region:$REGION" diff --git a/scripts/use-runners.sh b/scripts/use-runners.sh new file mode 100755 index 00000000..79953f7f --- /dev/null +++ b/scripts/use-runners.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +SCDIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")") +SCDIR=$(realpath $SCDIR) + +(return 0 2>/dev/null) && sourced=1 || sourced=0 +if (( sourced == 0 )); then + echo "This script must be invoked using: source $0 $*" + exit 1 +fi +export CLUSTER_NAME=stream-apps-gh-runners +set +e +PROVIDER=$($SCDIR/determine-provider.sh "$CLUSTER_NAME") +source $SCDIR/use-${PROVIDER}.sh $CLUSTER_NAME +source $SCDIR/kubeconfig-${PROVIDER}.sh $CLUSTER_NAME diff --git a/scripts/use-tmc.sh b/scripts/use-tmc.sh new file mode 100755 index 00000000..384e6f17 --- /dev/null +++ b/scripts/use-tmc.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +SCDIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")") +SCDIR=$(realpath $SCDIR) + +(return 0 2>/dev/null) && sourced=1 || sourced=0 +if (( sourced == 0 )); then + echo "This script must be invoked using: source $0 $*" + exit 1 +fi +function usage() { + echo "Usage is $0 " + if [ $sourced -eq 0 ]; then + exit 1 + else + return 1 + fi +} + +if [ "$1" == "" ]; then + usage +fi +export CLUSTER_NAME="$1" +tmc configure --management-cluster-name aws-hosted --provisioner-name scdf-provisioner +tmc cluster auth kubeconfig get $CLUSTER_NAME > /dev/null diff --git a/scripts/wait-deployment.sh b/scripts/wait-deployment.sh new file mode 100755 index 00000000..018d1f73 --- /dev/null +++ b/scripts/wait-deployment.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +set +e +if [ "$1" == "" ]; then + echo "Deployment name required" + exit 2 +fi +DEPLOYMENT=$1 +if [ "$2" != "" ]; then + NAMESPACE="--namespace $2" +else + NAMESPACE= +fi +echo "Checking deployment $DEPLOYMENT" +kubectl rollout status deployment "$DEPLOYMENT" $NAMESPACE +REPLICAS=$(kubectl get deployment "$DEPLOYMENT" $NAMESPACE --output json | jq '.status.readyReplicas') +echo "Deployment $DEPLOYMENT: Replicas: $REPLICAS" +RETRIES=40 +while [[ "$REPLICAS" == "" ]] || [[ $REPLICAS -eq 0 ]]; do + STATUS=$(kubectl get deployment "$DEPLOYMENT" $NAMESPACE --output json | jq '.status') + echo "Waiting for $DEPLOYMENT Status: $STATUS" + sleep 15 + REPLICAS=$(kubectl get deployment "$DEPLOYMENT" $NAMESPACE --output json | jq '.status.readyReplicas') + RC=$? + echo "Deployment $DEPLOYMENT: Replicas: $REPLICAS" + if [ "$RC" != "0" ]; then + STATUS=$(kubectl get deployment "$DEPLOYMENT" $NAMESPACE --output json | jq '.status') + echo "Error checking deployment $DEPLOYMENT $RC: $STATUS" + exit $RC + fi + if (( RETRIES <= 0 )); then + STATUS=$(kubectl get deployment "$DEPLOYMENT" $NAMESPACE --output json) + echo "Timeout checking deployment $DEPLOYMENT in $NAMESPACE: $STATUS" + exit 1 + fi + RETRIES=$(( RETRIES - 1 )) +done +echo "Deployment $DEPLOYMENT done" diff --git a/scripts/wait-for-cluster-gke.sh b/scripts/wait-for-cluster-gke.sh new file mode 100755 index 00000000..3c63867a --- /dev/null +++ b/scripts/wait-for-cluster-gke.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +set +e +WAIT_NODES=false +while [ "$1" != "" ]; do + case $1 in + "--nodes") + WAIT_NODES=true + ;; + *) + CLUSTER_NAME="$1" + ;; + esac + shift +done + +if [ "$CLUSTER_NAME" = "" ]; then + echo "CLUSTER_NAME not defined" + exit 1 +fi +REGION=$(gcloud container clusters list | grep -F "$CLUSTER_NAME" | awk '{print $2}') +if [ "$REGION" == "" ]; then + echo "Cannot find $CLUSTER_NAME" + exit 2 +fi + +echo "Waiting for $CLUSTER_NAME" + +STATUS=$(gcloud container clusters describe "$CLUSTER_NAME" --region $REGION --format json | jq '.status' | sed 's/"//g') +while [ "$STATUS" != "RUNNING" ]; do + echo "Waiting for $CLUSTER_NAME. Status: $STATUS" + sleep 30 + STATUS=$(gcloud container clusters describe "$CLUSTER_NAME" --region $REGION --format json | jq '.status' | sed 's/"//g') + RC=$? + if [ "$RC" != "0" ]; then + echo "Error reading cluster $CLUSTER_NAME: $RC: $STATUS" + exit $RC + fi +done +echo "Connected to $CLUSTER_NAME: Status=$STATUS" +if [ "$WAIT_NODES" == "true" ]; then + NODE_POOL=$(gcloud container node-pools list --region "$REGION" --cluster "$CLUSTER_NAME" "--format=table(name,status,config.tags)" | grep -F "$CLUSTER_NAME") + STATUS=$(echo "$NODE_POOL" | awk '{print $2}') + POOL=$(echo "$NODE_POOL" | awk '{print $1}') + while [ "$STATUS" != "RUNNING" ]; do + echo "Waiting for node pool $CLUSTER_NAME/$POOL. Status: $STATUS" + sleep 30 + NODE_POOL=$(gcloud container node-pools list --region "$REGION" --cluster "$CLUSTER_NAME" "--format=table(name,status,config.tags)" | grep -F "$CLUSTER_NAME") + RC=$? + POOL=$(echo "$NODE_POOL" | awk '{print $1}') + STATUS=$(echo "$NODE_POOL" | awk '{print $2}') + if [ "$RC" != "0" ]; then + echo "Error reading node pool $CLUSTER_NAME/$POOL: $RC: $STATUS" + exit $RC + fi + done + echo "Node pool $POOL - Status: $STATUS" +fi diff --git a/scripts/wait-for-cluster-tmc.sh b/scripts/wait-for-cluster-tmc.sh new file mode 100755 index 00000000..b695198d --- /dev/null +++ b/scripts/wait-for-cluster-tmc.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +set +e + +if [ "$1" != "" ]; then + export CLUSTER_NAME="$1" +fi + +if [ "$CLUSTER_NAME" = "" ]; then + echo "CLUSTER_NAME not defined" + exit 1 +fi + +echo "Waiting for $CLUSTER_NAME" + +STATUS=$(tmc cluster get "$CLUSTER_NAME" --output json | jq '.status.phase' | sed 's/"//g') +while [ "$STATUS" != "READY" ]; do + echo "Waiting for $CLUSTER_NAME. Status: $STATUS" + sleep 30 + STATUS=$(tmc cluster get "$CLUSTER_NAME" --output json | jq '.status.phase' | sed 's/"//g') + RC=$? + if [ "$RC" != "0" ]; then + MSG=$(tmc cluster get "$CLUSTER_NAME" --output json | jq '.status') + echo "Error reading cluster $CLUSTER_NAME: $RC: $MSG" + exit $RC + fi +done +echo "Checking $CLUSTER_NAME health" +AGENT_READY=$(tmc cluster get "$CLUSTER_NAME" --output json | jq '.status.conditions."Agent-READY"') +READY=$(echo "$AGENT_READY" | jq '.status' | sed 's/"//g') +while [ "$READY" != "TRUE" ]; do + echo "$CLUSTER_NAME Ready=$READY. $AGENT_READY" + sleep 30 + AGENT_READY=$(tmc cluster get "$CLUSTER_NAME" --output json | jq '.status.conditions."Agent-READY"') + READY=$(echo "$AGENT_READY" | jq '.status' | sed 's/"//g') + RC=$? + if [ "$RC" != "0" ]; then + echo "$CLUSTER_NAME not ready. $RC $AGENT_READY" + exit $RC + fi +done +echo "Connected to $CLUSTER_NAME: Ready=$READY" diff --git a/scripts/wait-k8s.sh b/scripts/wait-k8s.sh new file mode 100755 index 00000000..5231a48a --- /dev/null +++ b/scripts/wait-k8s.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +RETRIES=$1 +ERR_DIR="/tmp/$(dirname $0)" +ERR_FILE="/tmp/$0.err" +mkdir -p "$ERR_DIR" +shift +# shellcheck disable=SC2086 +CONDITION=$(kubectl wait $* 2> $ERR_FILE) +COUNT=$(echo "$CONDITION" | grep -c -F "condition met") +if [ $COUNT -gt 0 ]; then + echo "Condition met $COUNT times" + exit 1 +fi +RC=$? +while [ "$RC" != "0" ]; do + # shellcheck disable=SC2086 + if [ -f $ERR_FILE ]; then + cat "$ERR_FILE" + rm -f "$ERR_FILE" + fi + kubectl wait $* 2> "$ERR_FILE" + if [ $COUNT -gt 0 ]; then + echo "Condition met $COUNT times" + exit 1 + fi + RC=$? + RETRIES=$(( RETRIES - 1 )) + echo "RC:$RC, RETRIES=$RETRIES" + if [ $RETRIES -eq 0 ]; then + break; + fi +done +if [ "$RC" != "0" ] && [ -f "$ERR_FILE" ]; then + cat "$ERR_FILE" + rm -f "$ERR_FILE" + rm -rf "$ERR_DIR" +fi +exit $RC