[GHA] Cleanup and cache purge via cloudflare action

2024-07-12 14:25:06 -04:00
parent 5144d97b1c
commit 62dd211877
8 changed files with 32 additions and 157 deletions
--- a/.github/workflows/aggregate-distro-update-sites.yml
+++ b/.github/workflows/aggregate-distro-update-sites.yml
@@ -40,7 +40,7 @@ jobs:
          CLOUDFLARE_ZONE_ID: ${{ secrets.CLOUDFLARE_ZONE_ID }}
          CLOUDFLARE_CACHE_TOKEN: ${{ secrets.CLOUDFLARE_CACHE_TOKEN }}
        run: |
-          echo "Akamai S3 Aggregate Update Sites Creation..."
+          echo "S3 Aggregate Update Sites Creation..."
          dirs=`aws s3 ls s3://${AWS_S3_BUCKET}/release/TOOLS/sts4/update/${{ inputs.version }}/`
          echo "${dirs}"
          pattern='^e[0-9]+.[0-9]+/$'
@@ -76,15 +76,10 @@ jobs:
            fi
          done
          echo "invalid_urls=$invalid_urls" >> $GITHUB_OUTPUT
-
-  purge_cache:
-    needs: [ update-aggregate-update-sites ]
-    runs-on: [self-hosted]
-    steps:
-      - uses: ./.github/actions/akamai-purge-osx
+      - uses: ./.github/actions/cloudflare-purge
        name: Invalidate URLs
-        env:
-          EDGERC: ${{ secrets.EDGERC }}
        with:
-          command: invalidate
-          urls: ${{ needs.update-aggregate-update-sites.outputs.invalid_urls }}
+          urls: ${{ steps.gen-aggregate-sites.outputs.invalid_urls }}
+          cloudflare_zone_id: ${{ secrets.CLOUDFLARE_ZONE_ID }}
+          cloudflare_cache_token: ${{ secrets.CLOUDFLARE_CACHE_TOKEN }}
+
--- a/.github/workflows/aggregate-ls-extensions-update-site.yml
+++ b/.github/workflows/aggregate-ls-extensions-update-site.yml
@@ -32,11 +32,8 @@ jobs:
        run: echo "date=`echo $(($(date +%s%N) / 1000000))`" >> $GITHUB_OUTPUT
      - name: Generate and Upload Aggregate Update Sites Data
        id: gen-aggregate-sites
-        env:
-          CLOUDFLARE_ZONE_ID: ${{ secrets.CLOUDFLARE_ZONE_ID }}
-          CLOUDFLARE_CACHE_TOKEN: ${{ secrets.CLOUDFLARE_CACHE_TOKEN }}
        run: |
-          echo "Akamai S3 Aggregate Update Sites Creation..."
+          echo "S3 Aggregate Update Sites Creation..."
          invalid_urls=""
          site_url="${DOWNLOAD_URL_ROOT}/release/TOOLS/sts4-language-server-integrations/${{ inputs.version }}"
          ${{ github.workspace }}/.github/scripts/generate-composite-site-files.sh ${{ steps.timestamp.outputs.date }} $site_url
@@ -49,15 +46,9 @@ jobs:
          aws s3 cp ./p2.index s3://$AWS_S3_BUCKET/release/TOOLS/sts4-language-server-integrations/
          invalid_urls+="${DOWNLOAD_URL_ROOT}/release/TOOLS/sts4-language-server-integrations/p2.index"
          echo "invalid_urls=$invalid_urls" >> $GITHUB_OUTPUT
-
-  purge_cache:
-    needs: [ update-aggregate-update-sites ]
-    runs-on: [self-hosted]
-    steps:
-      - uses: ./.github/actions/akamai-purge-osx
+      - uses: ./.github/actions/cloudflare-purge
        name: Invalidate URLs
-        env:
-          EDGERC: ${{ secrets.EDGERC }}
        with:
-          command: invalidate
-          urls: ${{ needs.update-aggregate-update-sites.outputs.invalid_urls }}
+          urls: ${{ steps.gen-aggregate-sites.outputs.invalid_urls }}
+          cloudflare_zone_id: ${{ secrets.CLOUDFLARE_ZONE_ID }}
+          cloudflare_cache_token: ${{ secrets.CLOUDFLARE_CACHE_TOKEN }}
--- a/.github/workflows/akamai-test.yml
+++ b/.github/workflows/akamai-test.yml
@@ -1,70 +0,0 @@
-name: Test Akamai Upload
-
-on:
-  workflow_dispatch:
-
-jobs:
-
-  akamai-upload:
-    name: Upload to Akamai
-    runs-on: ubuntu-latest
-    steps:
-      - name: Gen file and upload
-        id: gen-upload
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.TOOLS_CLOUDGATE_ACCESS_KEY }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLS_CLOUDGATE_SECRET_KEY  }}
-          AWS_DEFAULT_REGION: us-east-1
-        run: |
-          role=arn:aws:iam::${{ secrets.TOOLS_CLOUDGATE_ACCOUNT_ID }}:role/${{ secrets.TOOLS_CLOUDGATE_USER }}
-          export $(printf "AWS_ACCESS_KEY_ID=%s AWS_SECRET_ACCESS_KEY=%s AWS_SESSION_TOKEN=%s" \
-          $(aws sts assume-role \
-          --role-arn $role \
-          --role-session-name gha-upload \
-          --query "Credentials.[AccessKeyId,SecretAccessKey,SessionToken]" \
-          --output text))
-          echo "AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" >> $GITHUB_ENV
-          echo "AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" >> $GITHUB_ENV
-          echo "AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN" >> $GITHUB_ENV
-          echo "AWS_DEFAULT_REGION=$AWS_DEFAULT_REGION" >> $GITHUB_ENV
-      - name: Test Environment
-        env:
-          AWS_DEFAULT_REGION: us-east-1
-        run: |
-          file="upload-test.txt"
-          echo 'Upload test file' > $file
-          cat $file
-          echo $AWS_SESSION_TOKEN
-          aws s3 mv ./$file s3://tools-spring-io/test-akamai/$file
-          aws s3 rm s3://tools-spring-io/test-akamai/$file
-          
-
-  akamai-upload-via-script:
-    name: Upload to Akamai via Script
-    needs: [ akamai-upload ]
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9
-        with:
-          sparse-checkout: |
-            .github
-      - name: Configure Cloudgate
-        uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a
-        with:
-          aws-access-key-id:  ${{ secrets.TOOLS_CLOUDGATE_ACCESS_KEY }}
-          aws-secret-access-key: ${{ secrets.TOOLS_CLOUDGATE_SECRET_KEY  }}
-          role-to-assume: arn:aws:iam::${{ secrets.TOOLS_CLOUDGATE_ACCOUNT_ID }}:role/${{ secrets.TOOLS_CLOUDGATE_USER }}
-          role-session-name: gha-upload-2
-          aws-region: us-east-1
-          role-duration-seconds: 900
-          role-skip-session-tagging: true
-      - name: Test Environment
-        env:
-          AWS_DEFAULT_REGION: us-east-1
-        run: |
-          file="upload-test.txt"
-          echo 'Upload test file' > $file
-          cat $file
-          echo $AWS_SESSION_TOKEN
-          aws s3 mv ./$file s3://tools-spring-io/test-akamai/$file
-          aws s3 rm s3://tools-spring-io/test-akamai/$file
--- a/.github/workflows/cloudflare-clear-cache-s3-dir.yml
+++ b/.github/workflows/cloudflare-clear-cache-s3-dir.yml
@@ -41,9 +41,6 @@ jobs:
            .github
      - uses: ./.github/actions/cloudflare-purge
        name: Invalidate Cache for URLs
-        env:
-          CLOUDFLARE_ZONE_ID: ${{ secrets.CLOUDFLARE_ZONE_ID }}
-          CLOUDFLARE_CACHE_TOKEN: ${{ secrets.CLOUDFLARE_CACHE_TOKEN }}
        with:
          urls: ${{ steps.prepare-urls.outputs.urls }}
          cloudflare_zone_id: ${{ secrets.CLOUDFLARE_ZONE_ID }}
--- a/.github/workflows/eclipse-ls-extensions-build.yml
+++ b/.github/workflows/eclipse-ls-extensions-build.yml
@@ -81,30 +81,27 @@ jobs:
          else
            ./mvnw --batch-mode clean install -P${{ inputs.eclipse_profile }} -P${{ inputs.build_type }} $sts4_ls_version_param -Pgitactions -Pgpg.sign -Dmaven.test.skip=true -Dhttpclient.retry-max=20 -Declipse.p2.mirrors=false -Dtycho.localArtifacts=ignore
          fi
-      - name: Upload P2 Repo to S3 and Akamai
-        id: upload-p2-to-akamai
+      - name: Upload P2 Repo to S3
+        id: upload-p2
        run: |
          p2_path=`cat ${{ github.workspace }}/eclipse-language-servers/org.springframework.tooling.ls.integration.repository/target/s3-p2-repo-dist-path.txt`
          echo "p2_path=$p2_path" >> $GITHUB_OUTPUT
          echo "P2 path: ${p2_path}"
          aws s3 rm s3://$AWS_S3_BUCKET/${p2_path}/ --recursive
          aws s3 cp ${{ github.workspace }}/eclipse-language-servers/org.springframework.tooling.ls.integration.repository/target/repository/ s3://$AWS_S3_BUCKET/${p2_path}/ --recursive --no-progress
-#      - name: Verify Uploaded P2 Repo
-#        id: verify-p2-on-akamai
-#        run: |
-#          p2_path=`cat ${{ github.workspace }}/eclipse-language-servers/org.springframework.tooling.ls.integration.repository/target/s3-p2-repo-dist-path.txt`
-#          echo "Verifying uploaded to Akamai files..."
-#          cd ${{ github.workspace }}/eclipse-language-servers/org.springframework.tooling.ls.integration.repository/target/repository/plugins
-#          for f in `ls -p | grep -v '/$'`
-#          do
-#            wget --no-verbose -O /dev/null https://cdn.spring.io/spring-tools/$p2_path/plugins/$f
-#          done
      - name: Compute invalid URLs
        if: ${{ inputs.build_type != 'snapshot' }}
        id: compute-invalid-urls
        run: |
-          invalid_urls=`aws s3 ls s3://$AWS_S3_BUCKET/${{ steps.upload-p2-to-akamai.outputs.p2_path }}/ --recursive | awk '{$1=$2=$3=""; print $0}' | sed -e 's/^[ \t]*/https:\/\/cdn.spring.io\/spring-tools\//' | paste -sd' ' -`
+          invalid_urls=`aws s3 ls s3://$AWS_S3_BUCKET/${{ steps.upload-p2.outputs.p2_path }}/ --recursive | awk '{$1=$2=$3=""; print $0}' | sed -e 's/^[ \t]*/https:\/\/cdn.spring.io\/spring-tools\//' | paste -sd' ' -`
          echo "invalid_urls=$invalid_urls" >> $GITHUB_OUTPUT
+      - uses: ./.github/actions/cloudflare-purge
+        if: ${{ inputs.build_type != 'snapshot' }}
+        name: Invalidate URLs
+        with:
+          urls: ${{ steps.compute-invalid-urls.outputs.invalid_urls }}
+          cloudflare_zone_id: ${{ secrets.CLOUDFLARE_ZONE_ID }}
+          cloudflare_cache_token: ${{ secrets.CLOUDFLARE_CACHE_TOKEN }}
      - name: Verify Eclipse LS Extensions on e433
        if: ${{ inputs.build_type == 'snapshot' }}
        run: |
@@ -115,19 +112,6 @@ jobs:
            ./mvnw --batch-mode clean package -Pe433 -P${{ inputs.build_type }} -Dhttpclient.retry-max=20 -Declipse.p2.mirrors=false -Dmaven.test.skip=true -Dtycho.localArtifacts=ignore
          fi

-  purge_cache:
-    needs: [ eclipse-language-servers-build ]
-    if: ${{ inputs.build_type != 'snapshot' }}
-    runs-on: [self-hosted]
-    steps:
-      - uses: ./.github/actions/akamai-purge-osx
-        name: Invalidate URLs
-        env:
-          EDGERC: ${{ secrets.EDGERC }}
-        with:
-          command: invalidate
-          urls: ${{ needs.eclipse-language-servers-build.outputs.invalid_urls }}
-
  notify-failure:
    needs: [ eclipse-language-servers-build, purge_cache ]
    if: ${{ always() && contains(needs.*.result, 'failure') }}
--- a/.github/workflows/gh-hosted-eclipse-distro-build.yml
+++ b/.github/workflows/gh-hosted-eclipse-distro-build.yml
@@ -121,9 +121,9 @@ jobs:
          echo "Processing S3 update..."
          dist_path=${{ steps.s3-paths.outputs.dist_path }}
          ls spring-tool-suite-4*linux*.tar.gz*
-          echo "Removing old Linux .tar.gz files from Akamai s3..."
+          echo "Removing old Linux .tar.gz files from s3..."
          aws s3 rm s3://$AWS_S3_BUCKET/$dist_path --recursive --exclude "*" --include "spring-tool-suite-4*linux*.tar.gz*" --exclude "*/*"
-          echo "Uploading new Linux .tar.gz files to Akamai s3..."
+          echo "Uploading new Linux .tar.gz files to s3..."
          aws s3 cp . s3://$AWS_S3_BUCKET/$dist_path --recursive --exclude "*" --include "spring-tool-suite-4*linux*.tar.gz*" --exclude "*/*" --no-progress
      - name: Update Nightly Distro Downloads page
        if: ${{ inputs.build_type == 'snapshot' }}
@@ -174,9 +174,9 @@ jobs:
          echo "Processing S3 update..."
          ls spring-tool-suite-4*win*.zip*
          ls spring-tool-suite-4*win*.self-extracting.jar*
-          echo "Removing old win zip and self extracting jar files from Akamai..."
+          echo "Removing old win zip and self extracting jar files from S3..."
          aws s3 rm s3://$AWS_S3_BUCKET/${dist_path} --recursive --exclude "*" --include "spring-tool-suite-4*win*.zip*" --include "spring-tool-suite-4*win*.self-extracting.jar*" --exclude "*/*"
-          echo "Uploading new win zip and self extracting jar files to Akamai..."
+          echo "Uploading new win zip and self extracting jar files to S3..."
          aws s3 mv . s3://$AWS_S3_BUCKET/${dist_path} --recursive --exclude "*" --include "spring-tool-suite-4*win*.zip*" --include "spring-tool-suite-4*win*.self-extracting.jar*" --exclude "*/*" --no-progress
      - name: Update Nightly Distro Downloads page
        if: ${{ inputs.build_type == 'snapshot' }}
@@ -270,9 +270,9 @@ jobs:
          dist_path=${{ needs.eclipse-distro-build.outputs.dist_path }}
          echo "Processing S3 update..."
          ls spring-tool-suite-4*macosx*.dmg*
-          echo "Removing old dmg files from Akamai..."
+          echo "Removing old dmg files from S3..."
          aws s3 rm s3://$AWS_S3_BUCKET/${dist_path} --recursive --exclude "*" --include "spring-tool-suite-4*macosx*.dmg*" --exclude "*/*"
-          echo "Uploading new dmg files to Akamai..."
+          echo "Uploading new dmg files to S3..."
          aws s3 mv . s3://$AWS_S3_BUCKET/${dist_path} --recursive --exclude "*" --include "spring-tool-suite-4*macosx*.dmg*" --exclude "*/*" --no-progress
      - name: Update Nightly Distro Downloads page
        if: ${{ inputs.build_type == 'snapshot' && always() }}
@@ -295,7 +295,7 @@ jobs:
        with:
          sparse-checkout: |
            .github
-      - name: Collect URLs on Akamai to Purge
+      - name: Collect URLs to Purge
        id: collect-urls
        run: |
          ESCAPED_REPLACE_URL=$(printf '%s\n' "$DOWNLOAD_URL_ROOT/" | sed -e 's/[\/&]/\\&/g')
--- a/.github/workflows/publish-vscode-extension.yml
+++ b/.github/workflows/publish-vscode-extension.yml
@@ -16,6 +16,9 @@ on:
        required: true
        type: string

+env:
+  DOWNLOAD_URL_ROOT: https://cdn.spring.io/spring-tools
+
 jobs:

  publish-release:
@@ -26,7 +29,7 @@ jobs:
        id: download-release
        run: |
          vsix_file=${{ inputs.extension-name }}-${{ inputs.version }}-${{ inputs.postfix }}.vsix
-          wget https://cdn.spring.io/spring-tools/vscode-extensions/${{ inputs.extension-name }}/${{ inputs.version }}/release/$vsix_file
+          wget $DOWNLOAD_URL_ROOT/release/vscode-extensions/${{ inputs.extension-name }}/${{ inputs.version }}/$vsix_file
      - name: Publish to VSCode Marketplace
        id: publish-vsce
        run: |
--- a/.github/workflows/test-r2-bucket.yml
+++ b/.github/workflows/test-r2-bucket.yml
@@ -1,25 +0,0 @@
-name: Test R2 Bucket
-
-on:
-  workflow_dispatch:
-
-jobs:
-
-  akamai-upload:
-    name: Test R2 Bucket
-    runs-on: ubuntu-latest
-    steps:
-      - name: R2 bucket upload
-        id: upload
-        env:
-          AWS_ACCESS_KEY_ID: ${{ secrets.R2_TEST_ACCESS_KEY }}
-          AWS_SECRET_ACCESS_KEY: ${{ secrets.R2_TEST_SECRET_KEY  }}
-          AWS_DEFAULT_REGION: us-east-1
-          AWS_ENDPOINT_URL_S3:  https://13b7012a66b58025c7f02b7c7c597d13.r2.cloudflarestorage.com
-        run: |
-          file="upload-test.txt"
-          echo 'Upload test file' > $file
-          cat $file
-          aws s3 mv ./$file s3://cdn-test-spring-io/sts-test/$file
-          aws s3 ls s3://cdn-test-spring-io/sts-test/ --recursive | awk '{$1=$2=$3=""; print $0}'
-          aws s3 rm s3://cdn-test-spring-io/sts-test/$file