From c9f9154b3b9d919f4bdb42e97096bd59f952b86b Mon Sep 17 00:00:00 2001 From: aboyko Date: Thu, 16 Nov 2023 01:11:39 -0500 Subject: [PATCH] GHA: Integrate Akamai cache invalidation into ls extensions build --- .../workflows/eclipse-ls-extensions-build.yml | 38 +++++++++++++++---- .../gh-hosted-eclipse-distro-build.yml | 31 +++++++-------- 2 files changed, 43 insertions(+), 26 deletions(-) diff --git a/.github/workflows/eclipse-ls-extensions-build.yml b/.github/workflows/eclipse-ls-extensions-build.yml index 1aecfeffe..7e07073ae 100644 --- a/.github/workflows/eclipse-ls-extensions-build.yml +++ b/.github/workflows/eclipse-ls-extensions-build.yml @@ -35,6 +35,8 @@ jobs: eclipse-language-servers-build: runs-on: macos-latest-xl + output: + invalid_urls: ${{ steps.compute-invalid-urls.outputs.invalid_urls }} steps: - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 with: @@ -88,21 +90,28 @@ jobs: role-duration-seconds: 900 role-skip-session-tagging: true - name: Upload P2 Repo to S3 and Akamai + id: upload-p2-to-akamai run: | - dist_path=`cat ${{ github.workspace }}/eclipse-language-servers/org.springframework.tooling.ls.integration.repository/target/s3-p2-repo-dist-path.txt` - echo "Dist path: ${dist_path}" - aws s3 rm s3://tools-spring-io/${dist_path}/ --recursive - aws s3 cp ${{ github.workspace }}/eclipse-language-servers/org.springframework.tooling.ls.integration.repository/target/repository/ s3://tools-spring-io/${dist_path}/ --recursive --no-progress - aws s3 rm s3://dist.springsource.com/${dist_path}/ --recursive - aws s3 cp ${{ github.workspace }}/eclipse-language-servers/org.springframework.tooling.ls.integration.repository/target/repository/ s3://dist.springsource.com/${dist_path}/ --recursive --acl public-read --no-progress + p2_path=`cat ${{ github.workspace }}/eclipse-language-servers/org.springframework.tooling.ls.integration.repository/target/s3-p2-repo-dist-path.txt` + echo "p2_path=$p2_path" >> $GITHUB_OUTPUT + echo "P2 path: ${p2_path}" + aws s3 rm s3://tools-spring-io/${p2_path}/ --recursive + aws s3 cp ${{ github.workspace }}/eclipse-language-servers/org.springframework.tooling.ls.integration.repository/target/repository/ s3://tools-spring-io/${p2_path}/ --recursive --no-progress + aws s3 rm s3://dist.springsource.com/${p2_path}/ --recursive + aws s3 cp ${{ github.workspace }}/eclipse-language-servers/org.springframework.tooling.ls.integration.repository/target/repository/ s3://dist.springsource.com/${p2_path}/ --recursive --acl public-read --no-progress + - name: Compute invalid URLs + if: ${{ inputs.build_type == 'snapshot' }} + id: compute-invalid-urls + run: | + invalid_urls=`aws s3 ls s3://tools-spring-io/${{ steps.upload-p2-to-akamai.outputs.p2_path }}/ --recursive | awk '{$1=$2=$3=""; print $0}' | sed -e 's/^[ \t]*/https:\/\/cdn.spring.io\/spring-tools\//' | paste -sd' ' -` + echo "invalid_urls=$invalid_urls" >> $GITHUB_OUTPUT - name: Clear S3 Caches if: ${{ inputs.build_type != 'snapshot' }} env: CLOUDFLARE_ZONE_ID: ${{ secrets.CLOUDFLARE_ZONE_ID }} CLOUDFLARE_CACHE_TOKEN: ${{ secrets.CLOUDFLARE_CACHE_TOKEN }} run: | - dist_path=`cat ${{ github.workspace }}/eclipse-language-servers/org.springframework.tooling.ls.integration.repository/target/s3-p2-repo-dist-path.txt` - ${{ github.workspace }}/.github/scripts/clear-s3-caches.sh $dist_path + ${{ github.workspace }}/.github/scripts/clear-s3-caches.sh ${{ steps.upload-p2-to-akamai.outputs.p2_path }} - name: Verify Eclipse LS Extensions on e430 if: ${{ inputs.build_type == 'snapshot' }} run: | @@ -113,6 +122,19 @@ jobs: ./mvnw --batch-mode clean package -Pe430 -P${{ inputs.build_type }} -Dhttpclient.retry-max=20 -Declipse.p2.mirrors=false -Dmaven.test.skip=true -Dtycho.localArtifacts=ignore fi + purge_cache: + needs: [ eclipse-language-servers-build ] + if: ${{ inputs.build_type == 'snapshot' }} + runs-on: [self-hosted, macOS] + steps: + - uses: ./.github/actions/akamai-purge-osx + name: Invalidate URLs + env: + EDGERC: ${{ secrets.EDGERC }} + with: + command: invalidate + urls: ${{ needs.eclipse-language-servers-build.outputs.invalid_urls }} + notify-failure: needs: [ eclipse-language-servers-build ] if: ${{ always() && contains(needs.*.result, 'failure') }} diff --git a/.github/workflows/gh-hosted-eclipse-distro-build.yml b/.github/workflows/gh-hosted-eclipse-distro-build.yml index 1b5aec891..e7d422f29 100644 --- a/.github/workflows/gh-hosted-eclipse-distro-build.yml +++ b/.github/workflows/gh-hosted-eclipse-distro-build.yml @@ -87,11 +87,12 @@ jobs: - name: Upload P2 Repo Build Artifacts run: | cd eclipse-distribution/org.springframework.boot.ide.product.${{ inputs.eclipse_profile }}/target - echo "P2 path: ${{ steps.s3-paths.outputs.p2_path }}" - aws s3 rm s3://dist.springsource.com/${{ steps.s3-paths.outputs.p2_path }}/ --recursive - aws s3 cp ./repository/ s3://dist.springsource.com/${{ steps.s3-paths.outputs.p2_path }}/ --recursive --acl public-read --no-progress - aws s3 rm s3://tools-spring-io/${{ steps.s3-paths.outputs.p2_path }}/ --recursive - aws s3 cp ./repository/ s3://tools-spring-io/${{ steps.s3-paths.outputs.p2_path }}/ --recursive --no-progress + p2_path=${{ steps.s3-paths.outputs.p2_path }} + echo "P2 path: ${p2_path}" + aws s3 rm s3://dist.springsource.com/$p2_path/ --recursive + aws s3 cp ./repository/ s3://dist.springsource.com/$p2_path/ --recursive --acl public-read --no-progress + aws s3 rm s3://tools-spring-io/$p2_path/ --recursive + aws s3 cp ./repository/ s3://tools-spring-io/$p2_path/ --recursive --no-progress - name: Upload Linux Distro Build Artifacts run: | cd eclipse-distribution/org.springframework.boot.ide.product.${{ inputs.eclipse_profile }}/target/products @@ -103,17 +104,18 @@ jobs: md5 $file > ${file}.md5 done echo "Processing S3 update..." + dist_path=${{ steps.s3-paths.outputs.dist_path }} ls spring-tool-suite-4*linux*.tar.gz* echo "Removing old Linux .tar.gz files from s3..." - aws s3 rm s3://dist.springsource.com/${{ steps.s3-paths.outputs.dist_path }} --recursive --exclude "*" --include "spring-tool-suite-4*linux*.tar.gz*" --exclude "*/*" + aws s3 rm s3://dist.springsource.com/$dist_path --recursive --exclude "*" --include "spring-tool-suite-4*linux*.tar.gz*" --exclude "*/*" echo "Uploading new Linux .tar.gz files to s3..." - aws s3 cp . s3://dist.springsource.com/${{ steps.s3-paths.outputs.dist_path }} --recursive --exclude "*" --include "spring-tool-suite-4*linux*.tar.gz*" --exclude "*/*" --acl public-read --no-progress + aws s3 cp . s3://dist.springsource.com/$dist_path --recursive --exclude "*" --include "spring-tool-suite-4*linux*.tar.gz*" --exclude "*/*" --acl public-read --no-progress echo "Removing old Linux .tar.gz files from Akamai s3..." - aws s3 rm s3://tools-spring-io/${{ steps.s3-paths.outputs.dist_path }} --recursive --exclude "*" --include "spring-tool-suite-4*linux*.tar.gz*" --exclude "*/*" + aws s3 rm s3://tools-spring-io/$dist_path --recursive --exclude "*" --include "spring-tool-suite-4*linux*.tar.gz*" --exclude "*/*" echo "Uploading new Linux .tar.gz files to Akamai s3..." - aws s3 cp . s3://tools-spring-io/${{ steps.s3-paths.outputs.dist_path }} --recursive --exclude "*" --include "spring-tool-suite-4*linux*.tar.gz*" --exclude "*/*" --no-progress + aws s3 cp . s3://tools-spring-io/$dist_path --recursive --exclude "*" --include "spring-tool-suite-4*linux*.tar.gz*" --exclude "*/*" --no-progress - name: Update Nightly Distro Downloads page - if: ${{ inputs.build_type == 'snapshot' && always() }} + if: ${{ inputs.build_type == 'snapshot' }} run: | eclipse_profile=${{ inputs.eclipse_profile }} eclipse_version=${eclipse_profile:0:2}.${eclipse_profile:2} @@ -123,13 +125,6 @@ jobs: echo "Uploading Win Zips and OSX tar.gz to S3 for signing..." id=${{ inputs.eclipse_profile }}-${{ inputs.build_type }}-${{ github.run_id }} aws s3 cp ${{ github.workspace }}/eclipse-distribution/org.springframework.boot.ide.product.${{ inputs.eclipse_profile }}/target/products s3://tools-spring-io/sts4-distro-ci-temp/$id --recursive --exclude "*" --include "spring-tool-suite-4*win*.zip" --include "spring-tool-suite-4*macosx*.tar.gz" --exclude "*/*" --no-progress - - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce - with: - name: s3-dist-path-${{ inputs.eclipse_profile }}-${{ inputs.build_type }}-${{ github.run_id }} - path: | - ${{ github.workspace }}/eclipse-distribution/org.springframework.boot.ide.product.${{ inputs.eclipse_profile }}/target/s3-dist-path.txt - ${{ github.workspace }}/eclipse-distribution/org.springframework.boot.ide.product.${{ inputs.eclipse_profile }}/target/s3-p2-repo-dist-path.txt - retention-days: 1 outputs: id: ${{ inputs.eclipse_profile }}-${{ inputs.build_type }}-${{ github.run_id }} dist_path: ${{ steps.s3-paths.outputs.dist_path }} @@ -301,7 +296,7 @@ jobs: purge_cache: needs: [ eclipse-distro-build, sign-win-distros, sign-osx-distros ] - if: ${{ inputs.build_type == 'snapshot' && contains(join(needs.*.result, ','), 'success')}} + if: ${{ always() && inputs.build_type == 'snapshot' && contains(join(needs.*.result, ','), 'success')}} runs-on: [self-hosted, macOS] steps: - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9