From 848bd5f52485b58d0f078e07c210c7078e3523ef Mon Sep 17 00:00:00 2001 From: aboyko Date: Thu, 9 Nov 2023 14:16:55 -0500 Subject: [PATCH] GHA: Use Cloudgate auth for s3 upload for distros --- .github/workflows/akamai-test.yml | 16 +-- .../workflows/eclipse-ls-extensions-build.yml | 4 +- .../gh-hosted-eclipse-distro-build.yml | 123 +++++++++++------- 3 files changed, 84 insertions(+), 59 deletions(-) diff --git a/.github/workflows/akamai-test.yml b/.github/workflows/akamai-test.yml index cf8f534d7..3fc747e15 100644 --- a/.github/workflows/akamai-test.yml +++ b/.github/workflows/akamai-test.yml @@ -47,20 +47,16 @@ jobs: ref: ${{ inputs.ref }} sparse-checkout: | .github - - name: Akamai Setup - id: akamai-setup + - name: Configure Cloudgate + uses: aws-actions/configure-aws-credentials@v4 env: AWS_ACCESS_KEY_ID: ${{ secrets.TOOLS_CLOUDGATE_ACCESS_KEY }} AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLS_CLOUDGATE_SECRET_KEY }} - TOOLS_CLOUDGATE_ACCOUNT_ID: ${{ secrets.TOOLS_CLOUDGATE_ACCOUNT_ID }} - TOOLS_CLOUDGATE_USER: ${{ secrets.TOOLS_CLOUDGATE_USER }} AWS_DEFAULT_REGION: us-east-1 - run: | - ${{ github.workspace }}/.github/scripts/akamai-aws-cli-env.sh gha-run-1 - echo "AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" >> $GITHUB_ENV - echo "AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" >> $GITHUB_ENV - echo "AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN" >> $GITHUB_ENV - echo "AWS_DEFAULT_REGION=$AWS_DEFAULT_REGION" >> $GITHUB_ENV + with: + role-to-assume: arn:aws:iam::${{ secrets.TOOLS_CLOUDGATE_ACCOUNT_ID }}:role/${{ secrets.TOOLS_CLOUDGATE_USER }} + role-session-name: gha-upload-2 + aws-region: us-east-1 - name: Test Environment env: AWS_DEFAULT_REGION: us-east-1 diff --git a/.github/workflows/eclipse-ls-extensions-build.yml b/.github/workflows/eclipse-ls-extensions-build.yml index cad452256..378fcee45 100644 --- a/.github/workflows/eclipse-ls-extensions-build.yml +++ b/.github/workflows/eclipse-ls-extensions-build.yml @@ -77,8 +77,8 @@ jobs: else ./mvnw --batch-mode clean install -P${{ inputs.eclipse_profile }} -P${{ inputs.build_type }} $sts4_ls_version_param -Pgitactions -Pgpg.sign -Dmaven.test.skip=true -Dhttpclient.retry-max=20 -Declipse.p2.mirrors=false -Dtycho.localArtifacts=ignore fi - - name: Akamai Setup - id: akamai-setup + - name: Cloudgate S3 Setup + id: cloudgate-setup env: AWS_ACCESS_KEY_ID: ${{ secrets.TOOLS_CLOUDGATE_ACCESS_KEY }} AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLS_CLOUDGATE_SECRET_KEY }} diff --git a/.github/workflows/gh-hosted-eclipse-distro-build.yml b/.github/workflows/gh-hosted-eclipse-distro-build.yml index 7722e9cc6..3011a52b4 100644 --- a/.github/workflows/gh-hosted-eclipse-distro-build.yml +++ b/.github/workflows/gh-hosted-eclipse-distro-build.yml @@ -52,8 +52,6 @@ jobs: echo "${{ secrets.GPG_PASSPHRASE }}" | gpg --batch --yes --passphrase-fd 0 --import gpg.asc - name: Build Eclipse Distro env: - tools_s3_access_key: ${{ secrets.TOOLS_S3_ACCESS_KEY }} - tools_s3_secret_key: ${{ secrets.TOOLS_S3_SECRET_KEY }} gpg_passphrase: ${{ secrets.GPG_PASSPHRASE }} gpg_keyname: ${{ secrets.GPG_KEYID }} run: | @@ -69,11 +67,26 @@ jobs: fi cd eclipse-distribution ./mvnw --batch-mode -U clean install -P${{ inputs.eclipse_profile }} -P${{ inputs.build_type }} $sts4_ls_version_param $p2_qualifier_param -Pgitactions -Pgpg.sign -Dmaven.repo.local=~/.m2/repository-signed -Dhttpclient.retry-max=20 -Dmaven.test.skip=true -Declipse.p2.mirrors=false -Dtycho.localArtifacts=ignore -Dp2.replaceQualifier=true -Dorg.eclipse.ecf.provider.filetransfer.httpclient.retrieve.readTimeout=1200000 -Dorg.eclipse.equinox.p2.transport.ecf.retry=5 -Dskip.eclipserun.proxies=false -Dtycho.equinox.resolver.uses=true - - name: Upload P2 Repo Build Artifacts + - name: Cloudgate S3 Setup + id: cloudgate-setup env: - AWS_ACCESS_KEY_ID: ${{ secrets.TOOLS_S3_ACCESS_KEY }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLS_S3_SECRET_KEY }} + AWS_ACCESS_KEY_ID: ${{ secrets.TOOLS_CLOUDGATE_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLS_CLOUDGATE_SECRET_KEY }} + TOOLS_CLOUDGATE_ACCOUNT_ID: ${{ secrets.TOOLS_CLOUDGATE_ACCOUNT_ID }} + TOOLS_CLOUDGATE_USER: ${{ secrets.TOOLS_CLOUDGATE_USER }} AWS_DEFAULT_REGION: us-east-1 + run: | + export $(printf "AWS_ACCESS_KEY_ID=%s AWS_SECRET_ACCESS_KEY=%s AWS_SESSION_TOKEN=%s" \ + $(aws sts assume-role \ + --role-arn arn:aws:iam::${{ secrets.TOOLS_CLOUDGATE_ACCOUNT_ID }}:role/${{ secrets.TOOLS_CLOUDGATE_USER }} \ + --role-session-name ${{ github.run_id }} \ + --query "Credentials.[AccessKeyId,SecretAccessKey,SessionToken]" \ + --output text)) + echo "AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" >> $GITHUB_ENV + echo "AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" >> $GITHUB_ENV + echo "AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN" >> $GITHUB_ENV + echo "AWS_DEFAULT_REGION=$AWS_DEFAULT_REGION" >> $GITHUB_ENV + - name: Upload P2 Repo Build Artifacts run: | cd eclipse-distribution/org.springframework.boot.ide.product.${{ inputs.eclipse_profile }}/target dist_path=`cat s3-p2-repo-dist-path.txt` @@ -81,10 +94,6 @@ jobs: aws s3 rm s3://dist.springsource.com/${dist_path}/ --recursive --dryrun aws s3 cp ./repository/ s3://dist.springsource.com/${dist_path}/ --recursive --acl public-read --dryrun - name: Upload Linux Distro Build Artifacts - env: - AWS_ACCESS_KEY_ID: ${{ secrets.TOOLS_S3_ACCESS_KEY }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLS_S3_SECRET_KEY }} - AWS_DEFAULT_REGION: us-east-1 run: | cd eclipse-distribution/org.springframework.boot.ide.product.${{ inputs.eclipse_profile }}/target/products files=`ls spring-tool-suite-4*linux*.tar.gz` @@ -104,9 +113,6 @@ jobs: - name: Clear S3 Caches for P2 Repo if: ${{ inputs.build_type != 'snapshot' }} env: - AWS_ACCESS_KEY_ID: ${{ secrets.TOOLS_S3_ACCESS_KEY }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLS_S3_SECRET_KEY }} - AWS_DEFAULT_REGION: us-east-1 CLOUDFLARE_ZONE_ID: ${{ secrets.CLOUDFLARE_ZONE_ID }} CLOUDFLARE_CACHE_TOKEN: ${{ secrets.CLOUDFLARE_CACHE_TOKEN }} run: | @@ -114,19 +120,12 @@ jobs: ${{ github.workspace }}/.github/scripts/clear-s3-caches.sh $dist_path - name: Update Nightly Distro Downloads page if: ${{ inputs.build_type == 'snapshot' && always() }} - env: - AWS_ACCESS_KEY_ID: ${{ secrets.TOOLS_S3_ACCESS_KEY }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLS_S3_SECRET_KEY }} - AWS_DEFAULT_REGION: us-east-1 run: | dist_path=`cat ${{ github.workspace }}/eclipse-distribution/org.springframework.boot.ide.product.${{ inputs.eclipse_profile }}/target/s3-dist-path.txt` eclipse_profile=${{ inputs.eclipse_profile }} eclipse_version=${eclipse_profile:0:2}.${eclipse_profile:2} ${{ github.workspace }}/.github/scripts/update-distro-download-page.sh $dist_path $eclipse_version ${{ inputs.build_type }} - name: Upload Build Artifacts for Signing - env: - AWS_ACCESS_KEY_ID: ${{ secrets.TOOLS_S3_ACCESS_KEY }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLS_S3_SECRET_KEY }} run: | echo "Uploading Win Zips and OSX tar.gz to S3 for signing..." id=${{ inputs.eclipse_profile }}-${{ inputs.build_type }}-${{ github.run_id }} @@ -148,10 +147,26 @@ jobs: with: sparse-checkout: | .github - - name: Download Win Zips from S3 for Signing + - name: Cloudgate S3 Setup + id: cloudgate-setup env: - AWS_ACCESS_KEY_ID: ${{ secrets.TOOLS_S3_ACCESS_KEY }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLS_S3_SECRET_KEY }} + AWS_ACCESS_KEY_ID: ${{ secrets.TOOLS_CLOUDGATE_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLS_CLOUDGATE_SECRET_KEY }} + TOOLS_CLOUDGATE_ACCOUNT_ID: ${{ secrets.TOOLS_CLOUDGATE_ACCOUNT_ID }} + TOOLS_CLOUDGATE_USER: ${{ secrets.TOOLS_CLOUDGATE_USER }} + AWS_DEFAULT_REGION: us-east-1 + run: | + export $(printf "AWS_ACCESS_KEY_ID=%s AWS_SECRET_ACCESS_KEY=%s AWS_SESSION_TOKEN=%s" \ + $(aws sts assume-role \ + --role-arn arn:aws:iam::${{ secrets.TOOLS_CLOUDGATE_ACCOUNT_ID }}:role/${{ secrets.TOOLS_CLOUDGATE_USER }} \ + --role-session-name ${{ github.run_id }} \ + --query "Credentials.[AccessKeyId,SecretAccessKey,SessionToken]" \ + --output text)) + echo "AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" >> $GITHUB_ENV + echo "AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" >> $GITHUB_ENV + echo "AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN" >> $GITHUB_ENV + echo "AWS_DEFAULT_REGION=$AWS_DEFAULT_REGION" >> $GITHUB_ENV + - name: Download Win Zips from S3 for Signing run: | rm -f spring-tool-suite-4*win*.zip* rm -f spring-tool-suite-4*win*.self-extracting.jar* @@ -173,9 +188,6 @@ jobs: name: s3-dist-path-${{ needs.eclipse-distro-build.outputs.id }} - name: Update Win zip/jar on S3 id: update-s3 - env: - AWS_ACCESS_KEY_ID: ${{ secrets.TOOLS_S3_ACCESS_KEY }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLS_S3_SECRET_KEY }} run: | dist_path=`cat s3-dist-path.txt` echo "Processing S3 update..." @@ -187,10 +199,6 @@ jobs: aws s3 mv . s3://dist.springsource.com/${dist_path} --recursive --exclude "*" --include "spring-tool-suite-4*win*.zip*" --include "spring-tool-suite-4*win*.self-extracting.jar*" --exclude "*/*" --acl public-read --no-progress - name: Update Nightly Distro Downloads page if: ${{ inputs.build_type == 'snapshot' && always() }} - env: - AWS_ACCESS_KEY_ID: ${{ secrets.TOOLS_S3_ACCESS_KEY }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLS_S3_SECRET_KEY }} - AWS_DEFAULT_REGION: us-east-1 run: | dist_path=`cat s3-dist-path.txt` eclipse_profile=${{ inputs.eclipse_profile }} @@ -237,10 +245,26 @@ jobs: run: | python3 -m pip install setuptools npm install -g appdmg - - name: Download OSX tar.gz files from S3 for Signing/Notarization + - name: Cloudgate S3 Setup + id: cloudgate-setup env: - AWS_ACCESS_KEY_ID: ${{ secrets.TOOLS_S3_ACCESS_KEY }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLS_S3_SECRET_KEY }} + AWS_ACCESS_KEY_ID: ${{ secrets.TOOLS_CLOUDGATE_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLS_CLOUDGATE_SECRET_KEY }} + TOOLS_CLOUDGATE_ACCOUNT_ID: ${{ secrets.TOOLS_CLOUDGATE_ACCOUNT_ID }} + TOOLS_CLOUDGATE_USER: ${{ secrets.TOOLS_CLOUDGATE_USER }} + AWS_DEFAULT_REGION: us-east-1 + run: | + export $(printf "AWS_ACCESS_KEY_ID=%s AWS_SECRET_ACCESS_KEY=%s AWS_SESSION_TOKEN=%s" \ + $(aws sts assume-role \ + --role-arn arn:aws:iam::${{ secrets.TOOLS_CLOUDGATE_ACCOUNT_ID }}:role/${{ secrets.TOOLS_CLOUDGATE_USER }} \ + --role-session-name ${{ github.run_id }} \ + --query "Credentials.[AccessKeyId,SecretAccessKey,SessionToken]" \ + --output text)) + echo "AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" >> $GITHUB_ENV + echo "AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" >> $GITHUB_ENV + echo "AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN" >> $GITHUB_ENV + echo "AWS_DEFAULT_REGION=$AWS_DEFAULT_REGION" >> $GITHUB_ENV + - name: Download OSX tar.gz files from S3 for Signing/Notarization run: | rm -rf *macosx* ls @@ -272,11 +296,8 @@ jobs: - uses: actions/download-artifact@cbed621e49e4c01b044d60f6c80ea4ed6328b281 with: name: s3-dist-path-${{ needs.eclipse-distro-build.outputs.id }} - - name: Update GMG files on S3 + - name: Update DMG files on S3 id: update-s3 - env: - AWS_ACCESS_KEY_ID: ${{ secrets.TOOLS_S3_ACCESS_KEY }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLS_S3_SECRET_KEY }} run: | dist_path=`cat s3-dist-path.txt` echo "Processing S3 update..." @@ -287,10 +308,6 @@ jobs: aws s3 mv . s3://dist.springsource.com/${dist_path} --recursive --exclude "*" --include "spring-tool-suite-4*macosx*.dmg*" --exclude "*/*" --acl public-read --no-progress - name: Update Nightly Distro Downloads page if: ${{ inputs.build_type == 'snapshot' && always() }} - env: - AWS_ACCESS_KEY_ID: ${{ secrets.TOOLS_S3_ACCESS_KEY }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLS_S3_SECRET_KEY }} - AWS_DEFAULT_REGION: us-east-1 run: | dist_path=`cat s3-dist-path.txt` eclipse_profile=${{ inputs.eclipse_profile }} @@ -313,12 +330,28 @@ jobs: - uses: actions/download-artifact@cbed621e49e4c01b044d60f6c80ea4ed6328b281 with: name: s3-dist-path-${{ needs.eclipse-distro-build.outputs.id }} + - name: Cloudgate S3 Setup + id: cloudgate-setup + env: + AWS_ACCESS_KEY_ID: ${{ secrets.TOOLS_CLOUDGATE_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLS_CLOUDGATE_SECRET_KEY }} + TOOLS_CLOUDGATE_ACCOUNT_ID: ${{ secrets.TOOLS_CLOUDGATE_ACCOUNT_ID }} + TOOLS_CLOUDGATE_USER: ${{ secrets.TOOLS_CLOUDGATE_USER }} + AWS_DEFAULT_REGION: us-east-1 + run: | + export $(printf "AWS_ACCESS_KEY_ID=%s AWS_SECRET_ACCESS_KEY=%s AWS_SESSION_TOKEN=%s" \ + $(aws sts assume-role \ + --role-arn arn:aws:iam::${{ secrets.TOOLS_CLOUDGATE_ACCOUNT_ID }}:role/${{ secrets.TOOLS_CLOUDGATE_USER }} \ + --role-session-name ${{ github.run_id }} \ + --query "Credentials.[AccessKeyId,SecretAccessKey,SessionToken]" \ + --output text)) + echo "AWS_ACCESS_KEY_ID=$AWS_ACCESS_KEY_ID" >> $GITHUB_ENV + echo "AWS_SECRET_ACCESS_KEY=$AWS_SECRET_ACCESS_KEY" >> $GITHUB_ENV + echo "AWS_SESSION_TOKEN=$AWS_SESSION_TOKEN" >> $GITHUB_ENV + echo "AWS_DEFAULT_REGION=$AWS_DEFAULT_REGION" >> $GITHUB_ENV - name: Clear S3 Caches for Distros if: ${{ inputs.build_type != 'snapshot' }} env: - AWS_ACCESS_KEY_ID: ${{ secrets.TOOLS_S3_ACCESS_KEY }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLS_S3_SECRET_KEY }} - AWS_DEFAULT_REGION: us-east-1 CLOUDFLARE_ZONE_ID: ${{ secrets.CLOUDFLARE_ZONE_ID }} CLOUDFLARE_CACHE_TOKEN: ${{ secrets.CLOUDFLARE_CACHE_TOKEN }} run: | @@ -326,10 +359,6 @@ jobs: ${{ github.workspace }}/.github/scripts/clear-s3-caches.sh $dist_path - name: Remove Temp Build Artifacts from S3 id: cleanup-s3-temp-storage - env: - AWS_ACCESS_KEY_ID: ${{ secrets.TOOLS_S3_ACCESS_KEY }} - AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLS_S3_SECRET_KEY }} - AWS_DEFAULT_REGION: us-east-1 run: | aws s3 rm s3://dist.springsource.com/sts4-distro-ci-temp --recursive --exclude "*" --include "${{ needs.eclipse-distro-build.outputs.id }}/*"