diff --git a/.github/workflows/clean-housekeeping.yml b/.github/workflows/clean-housekeeping.yml new file mode 100644 index 00000000..f52833ff --- /dev/null +++ b/.github/workflows/clean-housekeeping.yml @@ -0,0 +1,27 @@ +name: 'Housekeeping cleanup' + +on: + workflow_dispatch: + +jobs: + clean: + runs-on: ubuntu-latest + concurrency: + group: housekeeping + cancel-in-progress: false + steps: + - name: 'Configure: Checkout scdf-pro' + uses: actions/checkout@v3 + with: + ref: 'main' + - name: 'Action: Ensure scripts are executable' + shell: bash + run: find . -type f -name "*.sh" -exec chmod a+x '{}' \; + - name: 'Action: Delete cancelled housekeeping' + if: ${{ always() }} + shell: bash + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + ./scripts/delete-cancelled-housekeeping.sh + echo "::notice ::Deleted cancelled housekeeping jobs" diff --git a/.github/workflows/housekeeping.yml b/.github/workflows/housekeeping.yml new file mode 100644 index 00000000..13d8b78c --- /dev/null +++ b/.github/workflows/housekeeping.yml @@ -0,0 +1,408 @@ +name: 'Housekeeping' + +on: + workflow_dispatch: + inputs: + last_success: + description: 'Default is 90 - The number of minutes since last success required to run' + default: '120' + required: false + delete_clusters: + default: 'false' + description: 'true - Indicates that remaining ci clusters should be deleted' + required: false + verbose: + description: 'Increase verbosity of script outputs' + required: false + default: 'false' + schedule: + - cron: '*/30 * * * *' + +jobs: + housekeeping: + runs-on: ubuntu-latest + concurrency: + group: housekeeping + cancel-in-progress: false + steps: + - name: 'Configure: Checkout stream-applications' + uses: actions/checkout@v3 + with: + ref: 'main' + - name: 'Action: Ensure scripts are executable' + shell: bash + run: find . -type f -name "*.sh" -exec chmod a+x '{}' \; + - name: 'Check: Check for active workflows' + id: only_one + uses: ./.github/actions/is_there_only_one + with: + github_token: ${{ secrets.GITHUB_TOKEN }} + - name: 'Check: Needs to re-run' + if: ${{ env.ALLOW_HOUSEKEEPING == 'true' }} + shell: bash + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + LAST_SUCCESS=$(./scripts/last-workflow-status.sh "${{ github.workflow }}" success) + LAST_LIMIT=${{ inputs.last_success }} + if [ "$LAST_LIMIT" == "" ] || [ "$LAST_LIMIT" == "null" ]; then + LAST_LIMIT=120 + fi + if (( LAST_SUCCESS < LAST_LIMIT )); then + echo "::warning ::Housekeeping prevented, Last: $LAST_SUCCESS, Limit: $LAST_LIMIT" + echo "ALLOW_HOUSEKEEPING=false" >> $GITHUB_ENV + else + ./scripts/list-last-workflows.sh + LATEST_FAILED=$(jq 'map(select(.status == "completed" and .conclusion == "failure" and .event == "schedule")) | .[0] | .updated' sort_last.json | sed 's/\"//g') + NAME_FAILED=$(jq 'map(select(.status == "completed" and .conclusion == "failure" and .event == "schedule")) | .[0] | .name' sort_last.json | sed 's/\"//g') + if [ "$LATEST_FAILED" != "" ] && [ "$LATEST_FAILED" != "null" ]; then + LAST_SEC=$(date --date "$LATEST_FAILED" +%s) + CURRENT_SEC=$(date +%s) + DIFF_MIN=$(((CURRENT_SEC - LAST_SEC) / 60)) + echo "::notice ::Last failed: $NAME_FAILED, $DIFF_MIN minutes ago" + if ((DIFF_MIN < LAST_LIMIT)); then + echo "::warning ::Housekeeping prevented, scheduled: $NAME_FAILED failed: $LATEST_FAILED, Limit: $LAST_LIMIT" + echo "ALLOW_HOUSEKEEPING=false" >> $GITHUB_ENV + fi + fi + fi + - name: 'Action: Cancel when no housekeeping' + if: ${{ env.ALLOW_HOUSEKEEPING != 'true' }} + shell: bash + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh run cancel ${{ github.run_id }} + echo "::notice ::Cancelled" + outputs: + allow_housekeeping: ${{ env.ALLOW_HOUSEKEEPING }} + scale-down-runners: + if: ${{ needs.housekeeping.outputs.allow_housekeeping == 'true' }} + runs-on: ubuntu-latest + needs: [ housekeeping ] + concurrency: + group: stream-apps-gh-runners + cancel-in-progress: false + steps: + - name: 'Configure: Checkout stream-applications' + uses: actions/checkout@v3 + with: + ref: 'main' + - name: 'Action: Ensure scripts are executable' + shell: bash + run: find . -type f -name "*.sh" -exec chmod a+x '{}' \; + - name: 'Configure: Runners Cluster provider' + shell: bash + run: | + PROVIDER=$(./scripts/determine-provider.sh stream-apps-gh-runners) + echo "PROVIDER=$PROVIDER" >> $GITHUB_ENV + - name: 'Install: gcloud cli' + if: ${{ env.PROVIDER == 'gke' }} + uses: ./.github/actions/install-gcloud + - name: 'Action: gcloud auth' + if: ${{ env.PROVIDER == 'gke' }} + id: auth_gcloud + uses: 'google-github-actions/auth@v0' + with: + create_credentials_file: true + credentials_json: ${{ secrets.GCP_CRED_JSON }} + - name: 'Configure: Install TMC' + if: ${{ env.PROVIDER == 'tmc' }} + uses: ./.github/actions/install-tmc + - name: 'Action: Login to TMC' + if: ${{ env.PROVIDER == 'tmc' }} + uses: ./.github/actions/auth-tmc + with: + tmc_api_token: '${{ secrets.TMC_API_TOKEN }}' + - name: 'Install: Groovy' + uses: ./.github/actions/install-groovy + with: + version: 4.0.4 + - name: 'Configure: Connect to stream-apps-gh-runners' + shell: bash + env: + VERBOSE: ${{ inputs.verbose == 'true' && '--verbose' || '' }} + run: | + set +e + CLUSTER_NAME="stream-apps-gh-runners" + source "./scripts/use-${PROVIDER}.sh" $CLUSTER_NAME + RC=$? + if (( RC != 0 )); then + echo "::error ::Could not connect cluster on ${PROVIDER}" + exit 1 + fi + source ./scripts/kubeconfig-${PROVIDER}.sh + RC=$? + if (( RC != 0 )); then + echo "::error ::Could not connect to runners on ${PROVIDER}" + exit 1 + fi + echo "CLUSTER_NAME=$CLUSTER_NAME" >> $GITHUB_ENV + echo "KUBECONFIG=$KUBECONFIG" >> $GITHUB_ENV + - name: 'Action: Check re-run' + shell: bash + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + VERBOSE: ${{ inputs.verbose == 'true' && '--verbose' || '' }} + run: | + # check ci jobs failed less than 90 minutes ago + RE_RUN=$(./scripts/check-rerun-workflows.sh 120) + if [ "$RE_RUN" != "" ]; then + for run in $RE_RUN; do + ./scripts/re-run-workflow.sh $run + done + echo "RE_RUN=true" >> $GITHUB_ENV + exit 0 + else + echo "::notice ::No jobs targeted for re-run" + fi + - name: 'Action: Limit runners' + if: ${{ env.RE_RUN != 'true' }} + shell: bash + env: + VERBOSE: ${{ inputs.verbose == 'true' && '--verbose' || '' }} + run: | + set +e + ./scripts/limit-runners.sh 1 2 + RC=$? + if (( RC != 0 )); then + echo "::warning ::Could not limit runners on ${PROVIDER}" + echo "RECREATE=true" >> $GITHUB_ENV + exit 0 + fi + - name: 'Action: Scale down stream-apps-gh-runners' + if: ${{ env.RE_RUN != 'true' }} + shell: bash + env: + VERBOSE: ${{ inputs.verbose == 'true' && '--verbose' || '' }} + run: | + set +e + SCALE=$(./scripts/determine-default.sh stream-apps-gh-runners "scale_down") + ./scripts/scale-cluster-nodes.sh stream-apps-gh-runners --nodes $SCALE --max $SCALE --shrink + RC=$? + if ((RC != 0)); then + echo "RECREATE=true" >> $GITHUB_ENV + fi + outputs: + recreate: ${{ env.RECREATE }} + scale-down-cluster: + if: ${{ needs.housekeeping.outputs.allow_housekeeping == 'true' && needs.housekeeping.outputs.scale-down-matrix != null && needs.housekeeping.outputs.scale-down-matrix != '' }} + runs-on: ubuntu-latest + needs: [ housekeeping ] + strategy: + fail-fast: false + matrix: + cluster: ${{ fromJson(needs.housekeeping.outputs.scale-down-matrix) }} + concurrency: + group: ${{ matrix.cluster }} + cancel-in-progress: false + steps: + - name: 'Configure: Checkout stream-applications' + uses: actions/checkout@v3 + with: + ref: 'main' + - name: 'Ensure scripts are executable' + shell: bash + run: find . -type f -name "*.sh" -exec chmod a+x '{}' \; + - name: 'Configure: cluster provider' + shell: bash + run: | + PROVIDER=$(./scripts/determine-provider.sh ${{ matrix.cluster }}) + echo "PROVIDER=$PROVIDER" >> $GITHUB_ENV + - name: 'Install: gcloud cli' + if: ${{ env.PROVIDER == 'gke' }} + uses: ./.github/actions/install-gcloud + - name: 'Action: gcloud auth' + if: ${{ env.PROVIDER == 'gke' }} + id: auth_gcloud + uses: 'google-github-actions/auth@v0' + with: + create_credentials_file: true + credentials_json: ${{ secrets.GCP_CRED_JSON }} + - name: 'Configure: Install TMC' + if: ${{ env.PROVIDER == 'tmc' }} + uses: ./.github/actions/install-tmc + - name: 'Action: Login to TMC' + if: ${{ env.PROVIDER == 'tmc' }} + uses: ./.github/actions/auth-tmc + with: + tmc_api_token: '${{ secrets.TMC_API_TOKEN }}' + - name: 'Install: Groovy' + uses: ./.github/actions/install-groovy + with: + version: 4.0.4 + - name: 'Action: Uninstall infrastructure services in: ${{ matrix.cluster }} in ${{ env.REGION }} for ${{ matrix.jobs }}' + shell: bash + env: + VERBOSE: ${{ inputs.verbose && '--verbose' || '' }} + CLUSTER_NAME: ${{ matrix.cluster }} + # Add 2 for the infrastructure + run: | + source ./scripts/kubeconfig-${{ env.PROVIDER }}.sh + echo "::notice ::Deleting install infrastructure services on ${{ matrix.cluster }}" + ./scripts/delete-infra.sh + - name: 'Action: Reduce cluster ${{ matrix.cluster }}' + shell: bash + timeout-minutes: 20 + env: + VERBOSE: ${{ inputs.verbose == 'true' && '--verbose' || '' }} + run: | + SCALE=$(./scripts/determine-default.sh ${{ matrix.cluster }} "scale_down") + ./scripts/scale-cluster-nodes.sh ${{ matrix.cluster }} $VERBOSE --shrink --nodes $SCALE --max $SCALE + delete-clusters: + if: ${{ needs.housekeeping.outputs.allow_housekeeping == 'true' && needs.housekeeping.outputs.delete-matrix != null && needs.housekeeping.outputs.delete-matrix != '' }} + runs-on: ubuntu-latest + needs: [ housekeeping ] + strategy: + fail-fast: false + matrix: + cluster: ${{ fromJson(needs.housekeeping.outputs.delete-matrix) }} + concurrency: + group: ${{ matrix.cluster }} + cancel-in-progress: false + steps: + - name: 'Configure: Checkout stream-applications' + uses: actions/checkout@v3 + with: + ref: 'main' + - name: 'Ensure scripts are executable' + shell: bash + run: find . -type f -name "*.sh" -exec chmod a+x '{}' \; + - name: 'Configure: cluster provider' + shell: bash + run: | + PROVIDER=$(./scripts/determine-provider.sh ${{ matrix.cluster }}) + echo "PROVIDER=$PROVIDER" >> $GITHUB_ENV + - name: 'Install: gcloud cli' + if: ${{ env.PROVIDER == 'gke' }} + uses: ./.github/actions/install-gcloud + - name: 'Action: gcloud auth' + if: ${{ env.PROVIDER == 'gke' }} + id: auth_gcloud + uses: 'google-github-actions/auth@v0' + with: + create_credentials_file: true + credentials_json: ${{ secrets.GCP_CRED_JSON }} + - name: 'Configure: Install TMC' + if: ${{ env.PROVIDER == 'tmc' }} + uses: ./.github/actions/install-tmc + - name: 'Action: Login to TMC' + if: ${{ env.PROVIDER == 'tmc' }} + uses: ./.github/actions/auth-tmc + with: + tmc_api_token: '${{ secrets.TMC_API_TOKEN }}' + - name: 'Action: delete cluster ${{ matrix.cluster }} on ${{ env.PROVIDER }}' + shell: bash + env: + VERBOSE: ${{ inputs.verbose == 'true' && '--verbose' || '' }} + run: ./scripts/delete-cluster-${PROVIDER}.sh ${{ matrix.cluster }} + deleter-resources-aws: + if: ${{ needs.housekeeping.outputs.allow_housekeeping == 'true' }} + needs: + - housekeeping + runs-on: ubuntu-latest + steps: + - name: 'Configure: Checkout stream-applications' + uses: actions/checkout@v3 + with: + ref: 'main' + - name: Ensure scripts are executable + shell: bash + run: find . -type f -name "*.sh" -exec chmod a+x '{}' \; + - name: 'Configure: AWS Credentials' + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-1 + - name: 'Action: Delete Orphaned volumes' + shell: bash + env: + VERBOSE: ${{ inputs.verbose == 'true' && '--verbose' || '' }} + run: | + ./scripts/aws-query-delete-volumes.sh + ./aws-delete-volumes.sh + deleter-resources-gke: + if: ${{ needs.housekeeping.outputs.allow_housekeeping == 'true' }} + needs: + - housekeeping + runs-on: ubuntu-latest + steps: + - name: 'Configure: Checkout stream-applications' + uses: actions/checkout@v3 + with: + ref: 'main' + - name: Ensure scripts are executable + shell: bash + run: find . -type f -name "*.sh" -exec chmod a+x '{}' \; + - name: 'Install: gcloud cli' + uses: ./.github/actions/install-gcloud + - name: 'Action: gcloud auth' + id: auth_gcloud + uses: 'google-github-actions/auth@v0' + with: + create_credentials_file: true + credentials_json: ${{ secrets.GCP_CRED_JSON }} + - name: 'Action: Delete orphaned volumes from GKE' + shell: bash + env: + VERBOSE: ${{ inputs.verbose == 'true' && '--verbose' || '' }} + run: | + ./scripts/gke-query-delete-volumes.sh + ./gke-delete-volumes.sh + - name: 'Action: Delete orphaned forwarding rules from GKE' + shell: bash + env: + VERBOSE: ${{ inputs.verbose == 'true' && '--verbose' || '' }} + run: | + ./scripts/find-and-delete-gke-orphan-fw.sh + ./delete-orphaned-rules.sh + - name: 'Action: Delete orphaned target pools from GKE' + shell: bash + env: + VERBOSE: ${{ inputs.verbose == 'true' && '--verbose' || '' }} + run: | + ./scripts/find-and-delete-gke-orphan-tp.sh + ./delete-orphaned-pools.sh + recreate-runners: + if: ${{ needs.scale-down-runners.outputs.recreate == 'true' }} + needs: + - scale-down-runners + uses: ./.github/workflows/recreate-runners.yml + with: + verbose: ${{ inputs.verbose == 'true' && 'true' || 'false' }} + delete_cluster: 'true' + deployment_type: 'helm' + secrets: + TMC_API_TOKEN: ${{ secrets.TMC_API_TOKEN }} + GCP_CRED_JSON: ${{ secrets.GCP_CRED_JSON }} + DOCKER_HUB_USERNAME: ${{ secrets.DOCKER_HUB_USERNAME }} + DOCKER_HUB_PASSWORD: ${{ secrets.DOCKER_HUB_PASSWORD }} + GH_ARC_APP_ID: ${{ secrets.GH_ARC_APP_ID }} + GH_ARC_INSTALLATION_ID: ${{ secrets.GH_ARC_INSTALLATION_ID }} + GH_ARC_PRIVATE_KEY: ${{ secrets.GH_ARC_PRIVATE_KEY }} + cleanup: + if: ${{ always() }} + needs: + - recreate-runners + - deleter-resources-gke + - deleter-resources-aws + - scale-down-runners + - housekeeping + runs-on: ubuntu-latest + steps: + - name: 'Configure: Checkout stream-applications' + uses: actions/checkout@v3 + with: + ref: 'main' + - name: 'Action: Launch housekeeping cleanup' + shell: bash + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: gh workflow run clean-housekeeping.yml --json + - name: 'Action: Launch housekeeping on failure' + if: ${{ failure() }} + shell: bash + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: gh workflow run housekeeping.yml diff --git a/scripts/delete-cancelled-housekeeping.sh b/scripts/delete-cancelled-housekeeping.sh new file mode 100755 index 00000000..0d85c96a --- /dev/null +++ b/scripts/delete-cancelled-housekeeping.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +TO_DEL=$(gh run list --limit 100 -w "housekeeping.yml" --json "conclusion,event,databaseId,name,status,updatedAt" --jq 'map(select(.status == "completed" and .conclusion == "cancelled" and .event == "schedule"))') +if [ "$VERBOSE" != "" ] && [ "$VERBOSE" != "false" ]; then + echo "Deleting:$TO_DEL" +fi +RUNS=$(gh run list --limit 100 -w "housekeeping.yml" --json "conclusion,event,databaseId,name,status,updatedAt" --jq 'map(select(.status == "completed" and .conclusion == "cancelled" and .event == "schedule")) | .[] | .databaseId' | sed 's/\"//g') + +for run in $RUNS; do + echo "Deleting run $run" + gh api --method DELETE -H "Accept: application/vnd.github+json" "/repos/pivotal/scdf-pro/actions/runs/$run" +done + +RUNS=$(gh run list --limit 100 -w "clean-housekeeping.yml" --json "conclusion,databaseId,status" --jq 'map(select(.status == "completed" and .conclusion == "success")) | .[] | .databaseId' | sed 's/\"//g') +for run in $RUNS; do + echo "Deleting run $run" + gh api --method DELETE -H "Accept: application/vnd.github+json" "/repos/pivotal/scdf-pro/actions/runs/$run" +done diff --git a/scripts/last-workflow-status.sh b/scripts/last-workflow-status.sh new file mode 100755 index 00000000..79d4da74 --- /dev/null +++ b/scripts/last-workflow-status.sh @@ -0,0 +1,20 @@ +#!/usr/bin/bash +SCDIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")") +SCDIR=$(realpath $SCDIR) +if [ "$1" = "" ]; then + echo "Workflow name required" + exit 2 +fi +$SCDIR/list-last-workflows.sh "$1" success +if [ "$2" != "" ]; then + jq --arg conclusion "$2" '.[] | select(.conclusion == $conclusion)' sort_last.json | jq -s '.' > sort_last2.json + mv sort_last2.json sort_last.json +fi +LAST_TS=$(jq '.[0] | .updated' sort_last.json | sed 's/\"//g') +if [ "$LAST_TS" = "" ] || [ "$LAST_TS" = "null" ]; then + echo "0" +else + LAST_SEC=$(date --date "$LAST_TS" '+%s') + NOW_SEC=$(date '+%s') + echo "$(((NOW_SEC - LAST_SEC) / 60))" +fi diff --git a/scripts/list-last-workflows.sh b/scripts/list-last-workflows.sh index f473e436..f616abb6 100755 --- a/scripts/list-last-workflows.sh +++ b/scripts/list-last-workflows.sh @@ -1,23 +1,13 @@ #!/usr/bin/env bash -WORKFLOWS=("gke-full-ci.yml" "gke-ga-ci.yml" "gke-main-ci.yml" "gke-quick-ga-ci.yml" "gke-quick-main-ci.yml" "gke-smoke-ga-ci.yml" "gke-smoke-main-ci.yml") -echo "Listing last workflows" -echo "[" > last.json -ITEMS=0 +if [ "$1" != "" ]; then + WORKFLOWS=("$1") +else + WORKFLOWS=("ci-2021-1-x.yml" "ci-2021-0-x.yml" "ci-main.yml") +fi +echo "" > last.json for workflow in ${WORKFLOWS[@]}; do - WF_JSON=$(gh run list -w "$workflow" --limit 1 --json conclusion,event,databaseId,name,status,updatedAt) - CONCLUSION=$(echo "$WF_JSON" | jq '.[0].conclusion' | sed 's/\"//g') - if [ "$CONCLUSION" != "null" ] && [ "$CONCLUSION" != "" ]; then - STATUS=$(echo "$WF_JSON" | jq '.[0].status' | sed 's/\"//g') - UPDATED=$(echo "$WF_JSON" | jq '.[0].updatedAt' | sed 's/\"//g') - EVENT=$(echo "$WF_JSON" | jq '.[0].event' | sed 's/\"//g') - NAME=$(echo "$WF_JSON" | jq '.[0].name' | sed 's/\"//g') - ID=$(echo "$WF_JSON" | jq '.[0].databaseId') - if ((ITEMS > 0)); then - echo "," >> last.json - fi - ITEMS=$((ITEMS+1)) - echo "{\"updated\":\"$UPDATED\", \"status\":\"$STATUS\", \"conclusion\":\"$CONCLUSION\", \"event\":\"$EVENT\", \"id\":\"$ID\", \"name\":\"$NAME\"}" >> last.json - fi + rm -f wf.json + gh run list -w "$workflow" --json conclusion,event,databaseId,name,status,updatedAt > wf.json + jq -c '.[] | {"updated":.updatedAt,"status":.status,"conclusion":.conclusion,"event":.event,"id":.databaseId,"name":.name}' wf.json >> last.json done -echo "]" >> last.json -jq -s '.[] | sort_by(.updated) | reverse' last.json > sort_last.json +jq -s 'sort_by(.updated) | reverse' last.json > sort_last.json