From d18670667661615ad1a87c95a4ce3a4a66a8d0ef Mon Sep 17 00:00:00 2001 From: Martin Lippert Date: Mon, 24 Jul 2023 11:16:00 +0200 Subject: [PATCH 01/51] remove entries from internal memory store when deleting outdated cache files from disc --- .../vscode/boot/index/cache/IndexCacheOnDisc.java | 1 + .../index/cache/test/IndexCacheOnDiscTest.java | 14 ++++++++++++++ 2 files changed, 15 insertions(+) diff --git a/headless-services/spring-boot-language-server/src/main/java/org/springframework/ide/vscode/boot/index/cache/IndexCacheOnDisc.java b/headless-services/spring-boot-language-server/src/main/java/org/springframework/ide/vscode/boot/index/cache/IndexCacheOnDisc.java index dd0e296e5..ccf52520e 100644 --- a/headless-services/spring-boot-language-server/src/main/java/org/springframework/ide/vscode/boot/index/cache/IndexCacheOnDisc.java +++ b/headless-services/spring-boot-language-server/src/main/java/org/springframework/ide/vscode/boot/index/cache/IndexCacheOnDisc.java @@ -308,6 +308,7 @@ public class IndexCacheOnDisc implements IndexCache { && key.getIndexer().equals(cacheKey.getIndexer()) && key.getCategory().equals(cacheKey.getCategory())) { cacheFiles[i].delete(); + this.stores.remove(key); } // cleanup old cache files without category information (pre 4.19.1 release) else if (key != null && !key.equals(cacheKey) diff --git a/headless-services/spring-boot-language-server/src/test/java/org/springframework/ide/vscode/boot/index/cache/test/IndexCacheOnDiscTest.java b/headless-services/spring-boot-language-server/src/test/java/org/springframework/ide/vscode/boot/index/cache/test/IndexCacheOnDiscTest.java index aa4a99177..4a2c7d5a0 100644 --- a/headless-services/spring-boot-language-server/src/test/java/org/springframework/ide/vscode/boot/index/cache/test/IndexCacheOnDiscTest.java +++ b/headless-services/spring-boot-language-server/src/test/java/org/springframework/ide/vscode/boot/index/cache/test/IndexCacheOnDiscTest.java @@ -217,6 +217,20 @@ public class IndexCacheOnDiscTest { assertFalse(Files.exists(tempDir.resolve(Paths.get(key1.toString() + ".json")))); } + @Test + void testDoNotRetrieveOldCacheDataIfNewerVersionIsStored() throws Exception { + IndexCacheKey key1 = CACHE_KEY_VERSION_1; + IndexCacheKey key2 = new IndexCacheKey("someProject", "someIndexer", "someCategory", "2"); + + cache.store(key1, new String[0], new ArrayList<>(), null, CachedSymbol.class); + assertNotNull(cache.retrieve(key1, new String[0], CachedSymbol.class)); + assertNull(cache.retrieve(key2, new String[0], CachedSymbol.class)); + + cache.store(key2, new String[0], new ArrayList<>(), null, CachedSymbol.class); + assertNull(cache.retrieve(key1, new String[0], CachedSymbol.class)); + assertNotNull(cache.retrieve(key2, new String[0], CachedSymbol.class)); + } + @Test void testDeleteOldCacheFileFromPreviousReleasesIfNewOneIsStored() throws Exception { IndexCacheKey key1 = new IndexCacheKey("someProject", "someIndexer", "", "2"); From e21343ac83ab8fa2c05bb2def18be89d96186a90 Mon Sep 17 00:00:00 2001 From: aboyko Date: Fri, 28 Jul 2023 14:23:30 -0400 Subject: [PATCH 02/51] Check for classpath change is back Back to complete the revert of including tracking output folder contents into classpath changed event --- .../ide/vscode/boot/jdt/ls/JdtLsProjectCache.java | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/headless-services/spring-boot-language-server/src/main/java/org/springframework/ide/vscode/boot/jdt/ls/JdtLsProjectCache.java b/headless-services/spring-boot-language-server/src/main/java/org/springframework/ide/vscode/boot/jdt/ls/JdtLsProjectCache.java index 8f7501b6a..a550dae7e 100644 --- a/headless-services/spring-boot-language-server/src/main/java/org/springframework/ide/vscode/boot/jdt/ls/JdtLsProjectCache.java +++ b/headless-services/spring-boot-language-server/src/main/java/org/springframework/ide/vscode/boot/jdt/ls/JdtLsProjectCache.java @@ -360,6 +360,10 @@ public class JdtLsProjectCache implements InitializableJavaProjectsService, Serv URI projectUri = new URI(uri); ClasspathData classpath = new ClasspathData(event.name, event.classpath.getEntries()); IJavaProject oldProject = table.get(uri); + if (oldProject != null && classpath.equals(oldProject.getClasspath())) { + // nothing has changed + return; + } IProjectBuild projectBuild = from(event.projectBuild); IJavaProject newProject = IS_JANDEX_INDEX ? new JavaProject(getFileObserver(), projectUri, classpath, From d9134c407cb48db9e869c8837a1e5305dc346c61 Mon Sep 17 00:00:00 2001 From: aboyko Date: Fri, 28 Jul 2023 14:43:26 -0400 Subject: [PATCH 03/51] Adopt latest rewrite-spring --- .../java/rewrite/reconcile/HttpSecurityLamdaDslCodeAction.java | 3 ++- .../reconcile/ServerHttpSecurityLambdaDslCodeAction.java | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/headless-services/spring-boot-language-server/src/main/java/org/springframework/ide/vscode/boot/java/rewrite/reconcile/HttpSecurityLamdaDslCodeAction.java b/headless-services/spring-boot-language-server/src/main/java/org/springframework/ide/vscode/boot/java/rewrite/reconcile/HttpSecurityLamdaDslCodeAction.java index 14003b4b0..11b3a7930 100644 --- a/headless-services/spring-boot-language-server/src/main/java/org/springframework/ide/vscode/boot/java/rewrite/reconcile/HttpSecurityLamdaDslCodeAction.java +++ b/headless-services/spring-boot-language-server/src/main/java/org/springframework/ide/vscode/boot/java/rewrite/reconcile/HttpSecurityLamdaDslCodeAction.java @@ -16,6 +16,7 @@ import org.openrewrite.ExecutionContext; import org.openrewrite.SourceFile; import org.openrewrite.Tree; import org.openrewrite.java.JavaVisitor; +import org.openrewrite.java.spring.boot2.ConvertToSecurityDslVisitor; import org.openrewrite.java.spring.boot2.HttpSecurityLambdaDsl; import org.openrewrite.java.tree.J.MethodInvocation; import org.openrewrite.marker.Range; @@ -45,7 +46,7 @@ public class HttpSecurityLamdaDslCodeAction implements RecipeCodeActionDescripto @Override public MethodInvocation visitMethodInvocation(MethodInvocation method, ExecutionContext p) { - if (recipe.getVisitor().isApplicableTopLevelMethodInvocation(method)) { + if (((ConvertToSecurityDslVisitor) recipe.getVisitor()).isApplicableTopLevelMethodInvocation(method)) { // Don't step into the method any further String uri = getCursor().firstEnclosing(SourceFile.class).getSourcePath().toUri().toASCIIString(); FixAssistMarker marker = new FixAssistMarker(Tree.randomId(), getId()).withLabel(PROBLEM_LABEL) diff --git a/headless-services/spring-boot-language-server/src/main/java/org/springframework/ide/vscode/boot/java/rewrite/reconcile/ServerHttpSecurityLambdaDslCodeAction.java b/headless-services/spring-boot-language-server/src/main/java/org/springframework/ide/vscode/boot/java/rewrite/reconcile/ServerHttpSecurityLambdaDslCodeAction.java index 31a439e84..ba34d3cc6 100644 --- a/headless-services/spring-boot-language-server/src/main/java/org/springframework/ide/vscode/boot/java/rewrite/reconcile/ServerHttpSecurityLambdaDslCodeAction.java +++ b/headless-services/spring-boot-language-server/src/main/java/org/springframework/ide/vscode/boot/java/rewrite/reconcile/ServerHttpSecurityLambdaDslCodeAction.java @@ -16,6 +16,7 @@ import org.openrewrite.ExecutionContext; import org.openrewrite.SourceFile; import org.openrewrite.Tree; import org.openrewrite.java.JavaVisitor; +import org.openrewrite.java.spring.boot2.ConvertToSecurityDslVisitor; import org.openrewrite.java.spring.boot2.ServerHttpSecurityLambdaDsl; import org.openrewrite.java.tree.J.MethodInvocation; import org.openrewrite.marker.Range; @@ -45,7 +46,7 @@ public class ServerHttpSecurityLambdaDslCodeAction implements RecipeCodeActionDe @Override public MethodInvocation visitMethodInvocation(MethodInvocation method, ExecutionContext p) { - if (recipe.getVisitor().isApplicableTopLevelMethodInvocation(method)) { + if (((ConvertToSecurityDslVisitor)recipe.getVisitor()).isApplicableTopLevelMethodInvocation(method)) { // Don't step into the method any further String uri = getCursor().firstEnclosing(SourceFile.class).getSourcePath().toUri().toASCIIString(); FixAssistMarker marker = new FixAssistMarker(Tree.randomId(), getId()).withLabel(PROBLEM_LABEL) From 9ac817043e938bfebd6b924ce924a4de852e3ea6 Mon Sep 17 00:00:00 2001 From: aboyko Date: Sun, 30 Jul 2023 12:42:35 -0400 Subject: [PATCH 04/51] Preparation for eclipse ls extension release build --- .../workflows/eclipse-ls-extensions-build.yml | 34 ++++++++++++++----- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/.github/workflows/eclipse-ls-extensions-build.yml b/.github/workflows/eclipse-ls-extensions-build.yml index d39d69ff9..40580524c 100644 --- a/.github/workflows/eclipse-ls-extensions-build.yml +++ b/.github/workflows/eclipse-ls-extensions-build.yml @@ -11,6 +11,20 @@ on: required: true default: 'snapshot' type: string + eclipse_profile: + description: Eclipse profile 'e427' etc + required: false + default: 'e427' + type: string + skip_tests: + description: skip ls tests execution + required: false + default: false + type: boolean + sts4-language-servers-version: + description: version of sts4 ls extensions + required: false + type: string ref: description: Git branch, tag, commit hash required: false @@ -34,16 +48,13 @@ jobs: distribution: 'temurin' - name: Enforce https instead of http run: ./nohttp.sh - - name: Timestamp - id: timestamp - run: echo "name=date::$(date +'%Y%m%dT%H%M%S')" >> $GITHUB_OUTPUT - name: Build Language Servers JARs run: | cd headless-services if command -v xvfb-run ; then - xvfb-run ./mvnw --batch-mode clean install + xvfb-run ./mvnw --batch-mode clean install -DskipTests=${{ inputs.skip_tests }} else - ./mvnw --batch-mode clean install + ./mvnw --batch-mode clean install -DskipTests=${{ inputs.skip_tests }} fi - name: Install GPG key run: | @@ -56,17 +67,22 @@ jobs: gpg_passphrase: ${{ secrets.GPG_PASSPHRASE }} gpg_keyname: ${{ secrets.GPG_KEYID }} run: | + sts4_ls_version=${{ inputs.sts4-language-servers-version }} + sts4_ls_version_param="" + if ! [[ -z ${sts4_ls_version} ]]; then + sts4_ls_version_param="-Ddist.version=${sts4_ls_version}" + fi cd eclipse-language-servers if command -v xvfb-run ; then - xvfb-run ./mvnw --batch-mode clean deploy -Pe428 -P${{ inputs.build_type }} -Pgitactions -Pgpg.sign -DbuildNumber=${{ steps.timestamp.outputs.date }} -Dsigning.skip=true -Dhttpclient.retry-max=20 -Declipse.p2.mirrors=false -Ds3service.https-only=true -Dmaven.test.skip=true -Dtycho.localArtifacts=ignore + xvfb-run ./mvnw --batch-mode clean deploy -P{{ inputs.eclipse_profile }} -P${{ inputs.build_type }} ${sts4_ls_version_param} -Pgitactions -Pgpg.sign -Dmaven.test.skip=true -Dsigning.skip=true -Dhttpclient.retry-max=20 -Declipse.p2.mirrors=false -Ds3service.https-only=true -Dtycho.localArtifacts=ignore else - ./mvnw --batch-mode clean deploy -Pe428 -P${{ inputs.build_type }} -Pgitactions -Pgpg.sign -DbuildNumber=${{ steps.timestamp.outputs.date }} -Dsigning.skip=true -Dhttpclient.retry-max=20 -Declipse.p2.mirrors=false -Ds3service.https-only=true -Dmaven.test.skip=true -Dtycho.localArtifacts=ignore + ./mvnw --batch-mode clean deploy -P{{ inputs.eclipse_profile }} -P${{ inputs.build_type }} ${sts4_ls_version_param} -Pgitactions -Pgpg.sign -Dmaven.test.skip=true -Dsigning.skip=true -Dhttpclient.retry-max=20 -Declipse.p2.mirrors=false -Ds3service.https-only=true -Dtycho.localArtifacts=ignore fi - name: Verify Eclipse LS Extensions on e429 run: | cd eclipse-language-servers if command -v xvfb-run ; then - xvfb-run ./mvnw --batch-mode clean package -Pe429 -P${{ inputs.build_type }} -DbuildNumber=${{ steps.timestamp.outputs.timestamp }} -Dsigning.skip=true -Dhttpclient.retry-max=20 -Declipse.p2.mirrors=false -Ds3service.https-only=true -Dmaven.test.skip=true -Dtycho.localArtifacts=ignore + xvfb-run ./mvnw --batch-mode clean package -Pe429 -P${{ inputs.build_type }} -Dsigning.skip=true -Dhttpclient.retry-max=20 -Declipse.p2.mirrors=false -Ds3service.https-only=true -Dmaven.test.skip=true -Dtycho.localArtifacts=ignore else - ./mvnw --batch-mode clean package -Pe429 -P${{ inputs.build_type }} -DbuildNumber=${{ steps.timestamp.outputs.timestamp }} -Dsigning.skip=true -Dhttpclient.retry-max=20 -Declipse.p2.mirrors=false -Ds3service.https-only=true -Dmaven.test.skip=true -Dtycho.localArtifacts=ignore + ./mvnw --batch-mode clean package -Pe429 -P${{ inputs.build_type }} -Dsigning.skip=true -Dhttpclient.retry-max=20 -Declipse.p2.mirrors=false -Ds3service.https-only=true -Dmaven.test.skip=true -Dtycho.localArtifacts=ignore fi From f233cd7ae77f2a988fae845a60261053c613d09d Mon Sep 17 00:00:00 2001 From: aboyko Date: Sun, 30 Jul 2023 13:15:38 -0400 Subject: [PATCH 05/51] Preparation for eclipse distro release build --- .../workflows/eclipse-ls-extensions-build.yml | 4 +- .../gh-hosted-eclipse-distro-build.yml | 20 ++++++- .github/workflows/release-eclipse-build.yml | 57 +++++++++++++++++++ 3 files changed, 78 insertions(+), 3 deletions(-) create mode 100644 .github/workflows/release-eclipse-build.yml diff --git a/.github/workflows/eclipse-ls-extensions-build.yml b/.github/workflows/eclipse-ls-extensions-build.yml index 40580524c..a83be3d2b 100644 --- a/.github/workflows/eclipse-ls-extensions-build.yml +++ b/.github/workflows/eclipse-ls-extensions-build.yml @@ -74,9 +74,9 @@ jobs: fi cd eclipse-language-servers if command -v xvfb-run ; then - xvfb-run ./mvnw --batch-mode clean deploy -P{{ inputs.eclipse_profile }} -P${{ inputs.build_type }} ${sts4_ls_version_param} -Pgitactions -Pgpg.sign -Dmaven.test.skip=true -Dsigning.skip=true -Dhttpclient.retry-max=20 -Declipse.p2.mirrors=false -Ds3service.https-only=true -Dtycho.localArtifacts=ignore + xvfb-run ./mvnw --batch-mode clean deploy -P${{ inputs.eclipse_profile }} -P${{ inputs.build_type }} ${sts4_ls_version_param} -Pgitactions -Pgpg.sign -Dmaven.test.skip=true -Dsigning.skip=true -Dhttpclient.retry-max=20 -Declipse.p2.mirrors=false -Ds3service.https-only=true -Dtycho.localArtifacts=ignore else - ./mvnw --batch-mode clean deploy -P{{ inputs.eclipse_profile }} -P${{ inputs.build_type }} ${sts4_ls_version_param} -Pgitactions -Pgpg.sign -Dmaven.test.skip=true -Dsigning.skip=true -Dhttpclient.retry-max=20 -Declipse.p2.mirrors=false -Ds3service.https-only=true -Dtycho.localArtifacts=ignore + ./mvnw --batch-mode clean deploy -P${{ inputs.eclipse_profile }} -P${{ inputs.build_type }} ${sts4_ls_version_param} -Pgitactions -Pgpg.sign -Dmaven.test.skip=true -Dsigning.skip=true -Dhttpclient.retry-max=20 -Declipse.p2.mirrors=false -Ds3service.https-only=true -Dtycho.localArtifacts=ignore fi - name: Verify Eclipse LS Extensions on e429 run: | diff --git a/.github/workflows/gh-hosted-eclipse-distro-build.yml b/.github/workflows/gh-hosted-eclipse-distro-build.yml index 4baf37a7e..3bbdef0c0 100644 --- a/.github/workflows/gh-hosted-eclipse-distro-build.yml +++ b/.github/workflows/gh-hosted-eclipse-distro-build.yml @@ -13,6 +13,14 @@ on: required: true default: 'snapshot' type: string + sts4-language-servers-version: + description: version of sts4 ls extensions + required: false + type: string + p2-qualifier: + description: p2 qualifier to override + required: false + type: string ref: description: Git branch, tag, commit hash required: false @@ -49,8 +57,18 @@ jobs: gpg_passphrase: ${{ secrets.GPG_PASSPHRASE }} gpg_keyname: ${{ secrets.GPG_KEYID }} run: | + sts4_ls_version=${{ inputs.sts4-language-servers-version }} + sts4_ls_version_param="" + if ! [[ -z ${sts4_ls_version} ]]; then + sts4_ls_version_param="-Dsts4-language-servers-version=${sts4_ls_version}" + fi + p2_qualifier=${{ inputs.p2-qualifier }} + p2_qualifier_param="" + if ! [[ -z ${p2_qualifier} ]]; then + p2_qualifier_param="-Dp2.qualifier=${p2_qualifier}" + fi cd eclipse-distribution - ./mvnw --batch-mode -U clean deploy -P${{ inputs.eclipse_profile }} -P${{ inputs.build_type }} -Pgitactions -Pgpg.sign -Dsigning.skip=true -Dmaven.repo.local=~/.m2/repository-signed -Dhttpclient.retry-max=20 -Dmaven.test.skip=true -Declipse.p2.mirrors=false -Dtycho.localArtifacts=ignore -Ds3service.https-only=true -Dp2.replaceQualifier=true -Dorg.eclipse.ecf.provider.filetransfer.httpclient.retrieve.readTimeout=1200000 -Dorg.eclipse.equinox.p2.transport.ecf.retry=5 -Dskip.eclipserun.proxies=false -Dskip.osx.signing=true -Dskip.win.signing=true -Dskip.osx.notarizing=true -Dtycho.equinox.resolver.uses=true + ./mvnw --batch-mode -U clean deploy -P${{ inputs.eclipse_profile }} -P${{ inputs.build_type }} ${sts4_ls_version_param} ${p2_qualifier_param} -Pgitactions -Pgpg.sign -Dsigning.skip=true -Dmaven.repo.local=~/.m2/repository-signed -Dhttpclient.retry-max=20 -Dmaven.test.skip=true -Declipse.p2.mirrors=false -Dtycho.localArtifacts=ignore -Ds3service.https-only=true -Dp2.replaceQualifier=true -Dorg.eclipse.ecf.provider.filetransfer.httpclient.retrieve.readTimeout=1200000 -Dorg.eclipse.equinox.p2.transport.ecf.retry=5 -Dskip.eclipserun.proxies=false -Dskip.osx.signing=true -Dskip.win.signing=true -Dskip.osx.notarizing=true -Dtycho.equinox.resolver.uses=true - name: Upload Linux Distro Build Artifacts env: AWS_ACCESS_KEY_ID: ${{ secrets.TOOLS_S3_ACCESS_KEY }} diff --git a/.github/workflows/release-eclipse-build.yml b/.github/workflows/release-eclipse-build.yml new file mode 100644 index 000000000..541363542 --- /dev/null +++ b/.github/workflows/release-eclipse-build.yml @@ -0,0 +1,57 @@ +name: Release - Eclipse Build + +concurrency: + group: release-eclipse + cancel-in-progress: true + +on: + workflow_dispatch: + +jobs: + + eclipse-ls-extension: + runs-on: ubuntu-latest + steps: + - name: Timestamp + id: timestamp + run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT + - uses: ./.github/workflows/eclipse-ls-extensions-build.yml + with: + eclipse_profile: 'e427' + build_type: 'release' + skip_tests: true + sts4-language-servers-version: ${{ steps.timestamp.outputs.date }} + secrets: inherit + + e427-distro: + runs-on: ubuntu-latest + needs: [ eclipse-ls-extension ] + steps: + - uses: ./.github/workflows/gh-hosted-eclipse-distro-build.yml + with: + eclipse_profile: 'e427' + build_type: 'release' + p2-qualifier: 'RELEASE' + sts4-language-servers-version: ${{ jobs.eclipse-ls-extension.steps.timestamp.outputs.date }} + + e428-distro: + runs-on: ubuntu-latest + needs: [ eclipse-ls-extension ] + steps: + - uses: ./.github/workflows/gh-hosted-eclipse-distro-build.yml + with: + eclipse_profile: 'e428' + build_type: 'release' + p2-qualifier: 'RELEASE' + sts4-language-servers-version: ${{ jobs.eclipse-ls-extension.steps.timestamp.outputs.date }} + + e429-distro: + runs-on: ubuntu-latest + needs: [ eclipse-ls-extension ] + steps: + - uses: ./.github/workflows/gh-hosted-eclipse-distro-build.yml + with: + eclipse_profile: 'e429' + build_type: 'release' + p2-qualifier: 'RELEASE' + sts4-language-servers-version: ${{ jobs.eclipse-ls-extension.steps.timestamp.outputs.date }} From 657a255f97d1e047bee0ab7b6daca60cb5014753 Mon Sep 17 00:00:00 2001 From: aboyko Date: Sun, 30 Jul 2023 13:56:35 -0400 Subject: [PATCH 06/51] Corrections to preparation for eclipse distro release build --- .github/workflows/eclipse-ls-extensions-build.yml | 4 ++-- .github/workflows/gh-hosted-eclipse-distro-build.yml | 2 +- .github/workflows/release-eclipse-build.yml | 8 +++++--- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/eclipse-ls-extensions-build.yml b/.github/workflows/eclipse-ls-extensions-build.yml index a83be3d2b..9ad3b71c6 100644 --- a/.github/workflows/eclipse-ls-extensions-build.yml +++ b/.github/workflows/eclipse-ls-extensions-build.yml @@ -74,9 +74,9 @@ jobs: fi cd eclipse-language-servers if command -v xvfb-run ; then - xvfb-run ./mvnw --batch-mode clean deploy -P${{ inputs.eclipse_profile }} -P${{ inputs.build_type }} ${sts4_ls_version_param} -Pgitactions -Pgpg.sign -Dmaven.test.skip=true -Dsigning.skip=true -Dhttpclient.retry-max=20 -Declipse.p2.mirrors=false -Ds3service.https-only=true -Dtycho.localArtifacts=ignore + xvfb-run ./mvnw --batch-mode clean deploy -P${{ inputs.eclipse_profile }} -P${{ inputs.build_type }} $sts4_ls_version_param -Pgitactions -Pgpg.sign -Dmaven.test.skip=true -Dsigning.skip=true -Dhttpclient.retry-max=20 -Declipse.p2.mirrors=false -Ds3service.https-only=true -Dtycho.localArtifacts=ignore else - ./mvnw --batch-mode clean deploy -P${{ inputs.eclipse_profile }} -P${{ inputs.build_type }} ${sts4_ls_version_param} -Pgitactions -Pgpg.sign -Dmaven.test.skip=true -Dsigning.skip=true -Dhttpclient.retry-max=20 -Declipse.p2.mirrors=false -Ds3service.https-only=true -Dtycho.localArtifacts=ignore + ./mvnw --batch-mode clean deploy -P${{ inputs.eclipse_profile }} -P${{ inputs.build_type }} $sts4_ls_version_param -Pgitactions -Pgpg.sign -Dmaven.test.skip=true -Dsigning.skip=true -Dhttpclient.retry-max=20 -Declipse.p2.mirrors=false -Ds3service.https-only=true -Dtycho.localArtifacts=ignore fi - name: Verify Eclipse LS Extensions on e429 run: | diff --git a/.github/workflows/gh-hosted-eclipse-distro-build.yml b/.github/workflows/gh-hosted-eclipse-distro-build.yml index 3bbdef0c0..c533efffc 100644 --- a/.github/workflows/gh-hosted-eclipse-distro-build.yml +++ b/.github/workflows/gh-hosted-eclipse-distro-build.yml @@ -68,7 +68,7 @@ jobs: p2_qualifier_param="-Dp2.qualifier=${p2_qualifier}" fi cd eclipse-distribution - ./mvnw --batch-mode -U clean deploy -P${{ inputs.eclipse_profile }} -P${{ inputs.build_type }} ${sts4_ls_version_param} ${p2_qualifier_param} -Pgitactions -Pgpg.sign -Dsigning.skip=true -Dmaven.repo.local=~/.m2/repository-signed -Dhttpclient.retry-max=20 -Dmaven.test.skip=true -Declipse.p2.mirrors=false -Dtycho.localArtifacts=ignore -Ds3service.https-only=true -Dp2.replaceQualifier=true -Dorg.eclipse.ecf.provider.filetransfer.httpclient.retrieve.readTimeout=1200000 -Dorg.eclipse.equinox.p2.transport.ecf.retry=5 -Dskip.eclipserun.proxies=false -Dskip.osx.signing=true -Dskip.win.signing=true -Dskip.osx.notarizing=true -Dtycho.equinox.resolver.uses=true + ./mvnw --batch-mode -U clean deploy -P${{ inputs.eclipse_profile }} -P${{ inputs.build_type }} $sts4_ls_version_param $p2_qualifier_param -Pgitactions -Pgpg.sign -Dsigning.skip=true -Dmaven.repo.local=~/.m2/repository-signed -Dhttpclient.retry-max=20 -Dmaven.test.skip=true -Declipse.p2.mirrors=false -Dtycho.localArtifacts=ignore -Ds3service.https-only=true -Dp2.replaceQualifier=true -Dorg.eclipse.ecf.provider.filetransfer.httpclient.retrieve.readTimeout=1200000 -Dorg.eclipse.equinox.p2.transport.ecf.retry=5 -Dskip.eclipserun.proxies=false -Dskip.osx.signing=true -Dskip.win.signing=true -Dskip.osx.notarizing=true -Dtycho.equinox.resolver.uses=true - name: Upload Linux Distro Build Artifacts env: AWS_ACCESS_KEY_ID: ${{ secrets.TOOLS_S3_ACCESS_KEY }} diff --git a/.github/workflows/release-eclipse-build.yml b/.github/workflows/release-eclipse-build.yml index 541363542..04014b05b 100644 --- a/.github/workflows/release-eclipse-build.yml +++ b/.github/workflows/release-eclipse-build.yml @@ -22,6 +22,8 @@ jobs: skip_tests: true sts4-language-servers-version: ${{ steps.timestamp.outputs.date }} secrets: inherit + outputs: + sts4-language-servers-version: ${{ steps.timestamp.outputs.date }} e427-distro: runs-on: ubuntu-latest @@ -32,7 +34,7 @@ jobs: eclipse_profile: 'e427' build_type: 'release' p2-qualifier: 'RELEASE' - sts4-language-servers-version: ${{ jobs.eclipse-ls-extension.steps.timestamp.outputs.date }} + sts4-language-servers-version: ${{ needs.eclipse-ls-extension.outputs.sts4-language-servers-version }} e428-distro: runs-on: ubuntu-latest @@ -43,7 +45,7 @@ jobs: eclipse_profile: 'e428' build_type: 'release' p2-qualifier: 'RELEASE' - sts4-language-servers-version: ${{ jobs.eclipse-ls-extension.steps.timestamp.outputs.date }} + sts4-language-servers-version: ${{ needs.eclipse-ls-extension.outputs.sts4-language-servers-version }} e429-distro: runs-on: ubuntu-latest @@ -54,4 +56,4 @@ jobs: eclipse_profile: 'e429' build_type: 'release' p2-qualifier: 'RELEASE' - sts4-language-servers-version: ${{ jobs.eclipse-ls-extension.steps.timestamp.outputs.date }} + sts4-language-servers-version: ${{ needs.eclipse-ls-extension.outputs.sts4-language-servers-version }} From 44a46ec8a9af2088f679ed75db5edd0c53448ee3 Mon Sep 17 00:00:00 2001 From: aboyko Date: Mon, 31 Jul 2023 14:48:59 -0400 Subject: [PATCH 07/51] [GHA-Build] Separate distro and ls extensions builds. Initial trial of clearing S3 caches --- .github/scripts/clear-s3-caches.sh | 12 ++++ .github/workflows/clear_s3_cache.yml | 26 ++++++++ .github/workflows/release-eclipse-build.yml | 59 ------------------- .../release-eclipse-distro-build.yml | 45 ++++++++++++++ .../release-eclipse-ls-extensions.yml | 26 ++++++++ .../snapshot-eclipse-ls-extensions-build.yml | 10 ++-- 6 files changed, 114 insertions(+), 64 deletions(-) create mode 100755 .github/scripts/clear-s3-caches.sh create mode 100644 .github/workflows/clear_s3_cache.yml delete mode 100644 .github/workflows/release-eclipse-build.yml create mode 100644 .github/workflows/release-eclipse-distro-build.yml create mode 100644 .github/workflows/release-eclipse-ls-extensions.yml diff --git a/.github/scripts/clear-s3-caches.sh b/.github/scripts/clear-s3-caches.sh new file mode 100755 index 000000000..2dfc4ebb4 --- /dev/null +++ b/.github/scripts/clear-s3-caches.sh @@ -0,0 +1,12 @@ +s3_path=$1 +invalidation_json=`aws cloudfront create-invalidation --distribution-id ECAO9Q8651L8M --paths $s3_path | jq '.'` +invalidation_id=`echo $invalidation_json | jq -r '.Invalidation.Id'` +invalidation_status=`echo $invalidation_json | jq -r '.Invalidation.Status'` +echo "ID=${invalidation_id} Status=${invalidation_status}" +while [ $invalidation_status = "InProgress" ] +do + echo "Invalidation status: ${invalidation_status}" + sleep 3 + invalidation_status=`aws cloudfront get-invalidation --distribution-id ECAO9Q8651L8M --id $invalidation_id | jq -r '.Invalidation.Status'` +done +echo "Final invalidation status: ${invalidation_status}" diff --git a/.github/workflows/clear_s3_cache.yml b/.github/workflows/clear_s3_cache.yml new file mode 100644 index 000000000..f03b751d3 --- /dev/null +++ b/.github/workflows/clear_s3_cache.yml @@ -0,0 +1,26 @@ +name: Clear S3 Cache for path + +on: + workflow_dispatch: + inputs: + path: + description: Path in the S3 bucket dist.springsource.com without leading slash + required: true + type: string + +jobs: + + clear-s3-cache: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 + with: + ref: ${{ inputs.ref }} + sparse-checkout: | + .github + - name: Invalidate S3 Cloudfront Cache + env: + AWS_ACCESS_KEY_ID: ${{ secrets.TOOLS_S3_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLS_S3_SECRET_KEY }} + run: | + ${{ github.workspace }}/.github/scripts/clear-s3-caches.sh ${{ inputs.path }} diff --git a/.github/workflows/release-eclipse-build.yml b/.github/workflows/release-eclipse-build.yml deleted file mode 100644 index 04014b05b..000000000 --- a/.github/workflows/release-eclipse-build.yml +++ /dev/null @@ -1,59 +0,0 @@ -name: Release - Eclipse Build - -concurrency: - group: release-eclipse - cancel-in-progress: true - -on: - workflow_dispatch: - -jobs: - - eclipse-ls-extension: - runs-on: ubuntu-latest - steps: - - name: Timestamp - id: timestamp - run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT - - uses: ./.github/workflows/eclipse-ls-extensions-build.yml - with: - eclipse_profile: 'e427' - build_type: 'release' - skip_tests: true - sts4-language-servers-version: ${{ steps.timestamp.outputs.date }} - secrets: inherit - outputs: - sts4-language-servers-version: ${{ steps.timestamp.outputs.date }} - - e427-distro: - runs-on: ubuntu-latest - needs: [ eclipse-ls-extension ] - steps: - - uses: ./.github/workflows/gh-hosted-eclipse-distro-build.yml - with: - eclipse_profile: 'e427' - build_type: 'release' - p2-qualifier: 'RELEASE' - sts4-language-servers-version: ${{ needs.eclipse-ls-extension.outputs.sts4-language-servers-version }} - - e428-distro: - runs-on: ubuntu-latest - needs: [ eclipse-ls-extension ] - steps: - - uses: ./.github/workflows/gh-hosted-eclipse-distro-build.yml - with: - eclipse_profile: 'e428' - build_type: 'release' - p2-qualifier: 'RELEASE' - sts4-language-servers-version: ${{ needs.eclipse-ls-extension.outputs.sts4-language-servers-version }} - - e429-distro: - runs-on: ubuntu-latest - needs: [ eclipse-ls-extension ] - steps: - - uses: ./.github/workflows/gh-hosted-eclipse-distro-build.yml - with: - eclipse_profile: 'e429' - build_type: 'release' - p2-qualifier: 'RELEASE' - sts4-language-servers-version: ${{ needs.eclipse-ls-extension.outputs.sts4-language-servers-version }} diff --git a/.github/workflows/release-eclipse-distro-build.yml b/.github/workflows/release-eclipse-distro-build.yml new file mode 100644 index 000000000..c9c1913c8 --- /dev/null +++ b/.github/workflows/release-eclipse-distro-build.yml @@ -0,0 +1,45 @@ +name: Release - Eclipse Build + +concurrency: + group: release-eclipse + cancel-in-progress: true + +on: + workflow_dispatch: + inputs: + sts4-language-servers-version: + description: Last segments of an s3 path for p2 Repo for eclipse ls extensions + required: true + type: string + +jobs: + + e427-distro: + runs-on: ubuntu-latest + steps: + - uses: ./.github/workflows/gh-hosted-eclipse-distro-build.yml + with: + eclipse_profile: 'e427' + build_type: 'release' + p2-qualifier: 'RELEASE' + sts4-language-servers-version: ${{ inputs.sts4-language-servers-version }} + + e428-distro: + runs-on: ubuntu-latest + steps: + - uses: ./.github/workflows/gh-hosted-eclipse-distro-build.yml + with: + eclipse_profile: 'e428' + build_type: 'release' + p2-qualifier: 'RELEASE' + sts4-language-servers-version: ${{ inputs.sts4-language-servers-version }} + + e429-distro: + runs-on: ubuntu-latest + steps: + - uses: ./.github/workflows/gh-hosted-eclipse-distro-build.yml + with: + eclipse_profile: 'e429' + build_type: 'release' + p2-qualifier: 'RELEASE' + sts4-language-servers-version: ${{ inputs.sts4-language-servers-version }} diff --git a/.github/workflows/release-eclipse-ls-extensions.yml b/.github/workflows/release-eclipse-ls-extensions.yml new file mode 100644 index 000000000..9c39c0090 --- /dev/null +++ b/.github/workflows/release-eclipse-ls-extensions.yml @@ -0,0 +1,26 @@ +name: Release - Eclipse LS Extensions Build + +concurrency: + group: release-eclipse-ls-extensions + cancel-in-progress: true + +on: + workflow_dispatch: + +jobs: + + eclipse-ls-extension: + runs-on: ubuntu-latest + steps: + - name: Timestamp + id: timestamp + run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT + - uses: ./.github/workflows/eclipse-ls-extensions-build.yml + with: + eclipse_profile: 'e427' + build_type: 'release' + skip_tests: true + sts4-language-servers-version: ${{ steps.timestamp.outputs.date }} + secrets: inherit + outputs: + sts4-language-servers-version: ${{ steps.timestamp.outputs.date }} \ No newline at end of file diff --git a/.github/workflows/snapshot-eclipse-ls-extensions-build.yml b/.github/workflows/snapshot-eclipse-ls-extensions-build.yml index f398da152..c0307c642 100644 --- a/.github/workflows/snapshot-eclipse-ls-extensions-build.yml +++ b/.github/workflows/snapshot-eclipse-ls-extensions-build.yml @@ -4,12 +4,12 @@ concurrency: group: eclipse-ls-extension-snapshot cancel-in-progress: true -#on: -# workflow_dispatch: on: - push: - branches: - - 'main' + workflow_dispatch: +#on: +# push: +# branches: +# - 'main' jobs: eclipse-ls-extensions: From 4c79a9f01adba35dfffc51863a036ab88fe08820 Mon Sep 17 00:00:00 2001 From: aboyko Date: Mon, 31 Jul 2023 14:58:30 -0400 Subject: [PATCH 08/51] [GHA-Build] Correct generation of 'sts4-language-servers-version' parameter --- .../release-eclipse-ls-extensions.yml | 22 +++++++++++-------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/.github/workflows/release-eclipse-ls-extensions.yml b/.github/workflows/release-eclipse-ls-extensions.yml index 9c39c0090..a49faa935 100644 --- a/.github/workflows/release-eclipse-ls-extensions.yml +++ b/.github/workflows/release-eclipse-ls-extensions.yml @@ -9,18 +9,22 @@ on: jobs: - eclipse-ls-extension: + generate-sts4-ls-extensions-version: runs-on: ubuntu-latest steps: - name: Timestamp id: timestamp run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT - - uses: ./.github/workflows/eclipse-ls-extensions-build.yml - with: - eclipse_profile: 'e427' - build_type: 'release' - skip_tests: true - sts4-language-servers-version: ${{ steps.timestamp.outputs.date }} - secrets: inherit outputs: - sts4-language-servers-version: ${{ steps.timestamp.outputs.date }} \ No newline at end of file + sts4-language-servers-version: ${{ steps.timestamp.outputs.date }} + + eclipse-ls-extension: + needs: [ generate-sts4-ls-extensions-version ] + runs-on: ubuntu-latest + - uses: ./.github/workflows/eclipse-ls-extensions-build.yml + with: + eclipse_profile: 'e427' + build_type: 'release' + skip_tests: true + sts4-language-servers-version: ${{ needs.generate-sts4-ls-extensions-version.outputs.sts4-language-servers-version }} + secrets: inherit From b5b111dbb52d8ca2f28d25aa29ac2e0d493bc0e8 Mon Sep 17 00:00:00 2001 From: aboyko Date: Mon, 31 Jul 2023 15:02:15 -0400 Subject: [PATCH 09/51] [GHA-Build] Fix yaml syntax error --- .github/workflows/release-eclipse-ls-extensions.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/release-eclipse-ls-extensions.yml b/.github/workflows/release-eclipse-ls-extensions.yml index a49faa935..e68595223 100644 --- a/.github/workflows/release-eclipse-ls-extensions.yml +++ b/.github/workflows/release-eclipse-ls-extensions.yml @@ -20,8 +20,7 @@ jobs: eclipse-ls-extension: needs: [ generate-sts4-ls-extensions-version ] - runs-on: ubuntu-latest - - uses: ./.github/workflows/eclipse-ls-extensions-build.yml + uses: ./.github/workflows/eclipse-ls-extensions-build.yml with: eclipse_profile: 'e427' build_type: 'release' From 1b1380b7194b03c2f95c5173724816e2ccf2596f Mon Sep 17 00:00:00 2001 From: aboyko Date: Mon, 31 Jul 2023 15:43:45 -0400 Subject: [PATCH 10/51] [GHA-Build] Correct condition expression for clear cache script --- .github/scripts/clear-s3-caches.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/clear-s3-caches.sh b/.github/scripts/clear-s3-caches.sh index 2dfc4ebb4..829f1eba4 100755 --- a/.github/scripts/clear-s3-caches.sh +++ b/.github/scripts/clear-s3-caches.sh @@ -3,7 +3,7 @@ invalidation_json=`aws cloudfront create-invalidation --distribution-id ECAO9Q86 invalidation_id=`echo $invalidation_json | jq -r '.Invalidation.Id'` invalidation_status=`echo $invalidation_json | jq -r '.Invalidation.Status'` echo "ID=${invalidation_id} Status=${invalidation_status}" -while [ $invalidation_status = "InProgress" ] +while [ "${invalidation_status}" == "InProgress" ] do echo "Invalidation status: ${invalidation_status}" sleep 3 From 4f28b2c06e0da16b4b043c720f8992b1f497f7dd Mon Sep 17 00:00:00 2001 From: aboyko Date: Mon, 31 Jul 2023 15:47:12 -0400 Subject: [PATCH 11/51] [GHA-Build] debug clear cache script --- .github/scripts/clear-s3-caches.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/scripts/clear-s3-caches.sh b/.github/scripts/clear-s3-caches.sh index 829f1eba4..3ec9f9779 100755 --- a/.github/scripts/clear-s3-caches.sh +++ b/.github/scripts/clear-s3-caches.sh @@ -1,5 +1,6 @@ s3_path=$1 -invalidation_json=`aws cloudfront create-invalidation --distribution-id ECAO9Q8651L8M --paths $s3_path | jq '.'` +invalidation_json=`aws cloudfront create-invalidation --distribution-id ECAO9Q8651L8M --paths $s3_path` +echo "Invalidation response: ${invalidation_json}" invalidation_id=`echo $invalidation_json | jq -r '.Invalidation.Id'` invalidation_status=`echo $invalidation_json | jq -r '.Invalidation.Status'` echo "ID=${invalidation_id} Status=${invalidation_status}" From ace0898259e2cbf1e4dee1c170af086ff555afcf Mon Sep 17 00:00:00 2001 From: aboyko Date: Mon, 31 Jul 2023 15:53:25 -0400 Subject: [PATCH 12/51] [GHA-Build] debug clear cache script --- .github/scripts/clear-s3-caches.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/scripts/clear-s3-caches.sh b/.github/scripts/clear-s3-caches.sh index 3ec9f9779..9ffdd7891 100755 --- a/.github/scripts/clear-s3-caches.sh +++ b/.github/scripts/clear-s3-caches.sh @@ -1,5 +1,5 @@ s3_path=$1 -invalidation_json=`aws cloudfront create-invalidation --distribution-id ECAO9Q8651L8M --paths $s3_path` +invalidation_json=`aws cloudfront create-invalidation --distribution-id ECAO9Q8651L8M --output json --paths "${s3_path}"` echo "Invalidation response: ${invalidation_json}" invalidation_id=`echo $invalidation_json | jq -r '.Invalidation.Id'` invalidation_status=`echo $invalidation_json | jq -r '.Invalidation.Status'` @@ -8,6 +8,6 @@ while [ "${invalidation_status}" == "InProgress" ] do echo "Invalidation status: ${invalidation_status}" sleep 3 - invalidation_status=`aws cloudfront get-invalidation --distribution-id ECAO9Q8651L8M --id $invalidation_id | jq -r '.Invalidation.Status'` + invalidation_status=`aws cloudfront get-invalidation --distribution-id ECAO9Q8651L8M --id $invalidation_id --output json | jq -r '.Invalidation.Status'` done echo "Final invalidation status: ${invalidation_status}" From 2c7043b783a61b268842a061631c755cf024008f Mon Sep 17 00:00:00 2001 From: aboyko Date: Mon, 31 Jul 2023 15:56:20 -0400 Subject: [PATCH 13/51] [GHA-Build] debug clear cache script --- .github/scripts/clear-s3-caches.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/scripts/clear-s3-caches.sh b/.github/scripts/clear-s3-caches.sh index 9ffdd7891..42e5996c2 100755 --- a/.github/scripts/clear-s3-caches.sh +++ b/.github/scripts/clear-s3-caches.sh @@ -1,4 +1,5 @@ s3_path=$1 +aws cloudfront create-invalidation --distribution-id ECAO9Q8651L8M --output json --paths "'${s3_path}'" invalidation_json=`aws cloudfront create-invalidation --distribution-id ECAO9Q8651L8M --output json --paths "${s3_path}"` echo "Invalidation response: ${invalidation_json}" invalidation_id=`echo $invalidation_json | jq -r '.Invalidation.Id'` From 4a7e99d2f91d3bb92d44b2fd7bf8d8ca154919d3 Mon Sep 17 00:00:00 2001 From: aboyko Date: Mon, 31 Jul 2023 16:00:30 -0400 Subject: [PATCH 14/51] [GHA-Build] debug clear cache script --- .github/scripts/clear-s3-caches.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/scripts/clear-s3-caches.sh b/.github/scripts/clear-s3-caches.sh index 42e5996c2..9cc12973c 100755 --- a/.github/scripts/clear-s3-caches.sh +++ b/.github/scripts/clear-s3-caches.sh @@ -1,6 +1,6 @@ s3_path=$1 -aws cloudfront create-invalidation --distribution-id ECAO9Q8651L8M --output json --paths "'${s3_path}'" -invalidation_json=`aws cloudfront create-invalidation --distribution-id ECAO9Q8651L8M --output json --paths "${s3_path}"` +aws cloudfront create-invalidation --distribution-id ECAO9Q8651L8M --output json --paths "${s3_path}/*" +invalidation_json=`aws cloudfront create-invalidation --distribution-id ECAO9Q8651L8M --output json --paths "${s3_path}/*"` echo "Invalidation response: ${invalidation_json}" invalidation_id=`echo $invalidation_json | jq -r '.Invalidation.Id'` invalidation_status=`echo $invalidation_json | jq -r '.Invalidation.Status'` From 25318dc3b96e0bb3415fb07e8f28347c242ad102 Mon Sep 17 00:00:00 2001 From: aboyko Date: Mon, 31 Jul 2023 16:02:07 -0400 Subject: [PATCH 15/51] [GHA-Build] debug clear cache script --- .github/scripts/clear-s3-caches.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/clear-s3-caches.sh b/.github/scripts/clear-s3-caches.sh index 9cc12973c..52858298f 100755 --- a/.github/scripts/clear-s3-caches.sh +++ b/.github/scripts/clear-s3-caches.sh @@ -1,5 +1,5 @@ s3_path=$1 -aws cloudfront create-invalidation --distribution-id ECAO9Q8651L8M --output json --paths "${s3_path}/*" +aws cloudfront create-invalidation --distribution-id ECAO9Q8651L8M --output json --paths 'release/TOOLS/sts4-language-server-integrations/2023-07-31/*' invalidation_json=`aws cloudfront create-invalidation --distribution-id ECAO9Q8651L8M --output json --paths "${s3_path}/*"` echo "Invalidation response: ${invalidation_json}" invalidation_id=`echo $invalidation_json | jq -r '.Invalidation.Id'` From 3aa349928d7d978969f3a8f2749268962bfe83bf Mon Sep 17 00:00:00 2001 From: aboyko Date: Mon, 31 Jul 2023 16:11:40 -0400 Subject: [PATCH 16/51] [GHA-Build] debug clear cache script --- .github/scripts/clear-s3-caches.sh | 4 ++-- .github/workflows/clear_s3_cache.yml | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/scripts/clear-s3-caches.sh b/.github/scripts/clear-s3-caches.sh index 52858298f..7f5283b05 100755 --- a/.github/scripts/clear-s3-caches.sh +++ b/.github/scripts/clear-s3-caches.sh @@ -1,6 +1,6 @@ s3_path=$1 -aws cloudfront create-invalidation --distribution-id ECAO9Q8651L8M --output json --paths 'release/TOOLS/sts4-language-server-integrations/2023-07-31/*' -invalidation_json=`aws cloudfront create-invalidation --distribution-id ECAO9Q8651L8M --output json --paths "${s3_path}/*"` +aws cloudfront create-invalidation --distribution-id ECAO9Q8651L8M --paths '/release/TOOLS/sts4-language-server-integrations/2023-07-31/*' +invalidation_json=`aws cloudfront create-invalidation --distribution-id ECAO9Q8651L8M --output json --paths "/${s3_path}/*"` echo "Invalidation response: ${invalidation_json}" invalidation_id=`echo $invalidation_json | jq -r '.Invalidation.Id'` invalidation_status=`echo $invalidation_json | jq -r '.Invalidation.Status'` diff --git a/.github/workflows/clear_s3_cache.yml b/.github/workflows/clear_s3_cache.yml index f03b751d3..ce1043218 100644 --- a/.github/workflows/clear_s3_cache.yml +++ b/.github/workflows/clear_s3_cache.yml @@ -23,4 +23,5 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.TOOLS_S3_ACCESS_KEY }} AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLS_S3_SECRET_KEY }} run: | + aws cloudfront create-invalidation --distribution-id ECAO9Q8651L8M --paths '/release/TOOLS/sts4-language-server-integrations/2023-07-31/*' ${{ github.workspace }}/.github/scripts/clear-s3-caches.sh ${{ inputs.path }} From bb21f957ede1f04aa1bfce996082f168390cf30f Mon Sep 17 00:00:00 2001 From: aboyko Date: Mon, 31 Jul 2023 16:18:15 -0400 Subject: [PATCH 17/51] [GHA-Build] debug clear cache script --- .github/scripts/clear-s3-caches.sh | 4 ++-- .github/workflows/clear_s3_cache.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/scripts/clear-s3-caches.sh b/.github/scripts/clear-s3-caches.sh index 7f5283b05..f42710fa3 100755 --- a/.github/scripts/clear-s3-caches.sh +++ b/.github/scripts/clear-s3-caches.sh @@ -1,6 +1,6 @@ s3_path=$1 -aws cloudfront create-invalidation --distribution-id ECAO9Q8651L8M --paths '/release/TOOLS/sts4-language-server-integrations/2023-07-31/*' -invalidation_json=`aws cloudfront create-invalidation --distribution-id ECAO9Q8651L8M --output json --paths "/${s3_path}/*"` +aws cloudfront create-invalidation --distribution-id ECAO9Q8651L8M --paths /release/TOOLS/sts4-language-server-integrations/2023-07-31/* +invalidation_json=`aws cloudfront create-invalidation --distribution-id ECAO9Q8651L8M --output json --paths /${s3_path}/*` echo "Invalidation response: ${invalidation_json}" invalidation_id=`echo $invalidation_json | jq -r '.Invalidation.Id'` invalidation_status=`echo $invalidation_json | jq -r '.Invalidation.Status'` diff --git a/.github/workflows/clear_s3_cache.yml b/.github/workflows/clear_s3_cache.yml index ce1043218..e514645a2 100644 --- a/.github/workflows/clear_s3_cache.yml +++ b/.github/workflows/clear_s3_cache.yml @@ -23,5 +23,5 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.TOOLS_S3_ACCESS_KEY }} AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLS_S3_SECRET_KEY }} run: | - aws cloudfront create-invalidation --distribution-id ECAO9Q8651L8M --paths '/release/TOOLS/sts4-language-server-integrations/2023-07-31/*' + aws cloudfront create-invalidation --distribution-id ECAO9Q8651L8M --paths /release/TOOLS/sts4-language-server-integrations/2023-07-31/* ${{ github.workspace }}/.github/scripts/clear-s3-caches.sh ${{ inputs.path }} From 263b7b1320aa1a0ff0f064aa05e66d360e1a091f Mon Sep 17 00:00:00 2001 From: aboyko Date: Mon, 31 Jul 2023 16:19:17 -0400 Subject: [PATCH 18/51] [GHA-Build] debug clear cache script --- .github/workflows/clear_s3_cache.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/clear_s3_cache.yml b/.github/workflows/clear_s3_cache.yml index e514645a2..36e34d9f6 100644 --- a/.github/workflows/clear_s3_cache.yml +++ b/.github/workflows/clear_s3_cache.yml @@ -24,4 +24,5 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLS_S3_SECRET_KEY }} run: | aws cloudfront create-invalidation --distribution-id ECAO9Q8651L8M --paths /release/TOOLS/sts4-language-server-integrations/2023-07-31/* + aws cloudfront list-invalidations --distribution-id ECAO9Q8651L8M ${{ github.workspace }}/.github/scripts/clear-s3-caches.sh ${{ inputs.path }} From 6441ae84eadd6c17f12fd3c2fcb6cf44f3851152 Mon Sep 17 00:00:00 2001 From: aboyko Date: Mon, 31 Jul 2023 16:21:12 -0400 Subject: [PATCH 19/51] [GHA-Build] debug clear cache script --- .github/workflows/clear_s3_cache.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/clear_s3_cache.yml b/.github/workflows/clear_s3_cache.yml index 36e34d9f6..f1560682c 100644 --- a/.github/workflows/clear_s3_cache.yml +++ b/.github/workflows/clear_s3_cache.yml @@ -23,6 +23,8 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.TOOLS_S3_ACCESS_KEY }} AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLS_S3_SECRET_KEY }} run: | - aws cloudfront create-invalidation --distribution-id ECAO9Q8651L8M --paths /release/TOOLS/sts4-language-server-integrations/2023-07-31/* + echo "Firing off invalidation request" + aws cloudfront create-invalidation --distribution-id ECAO9Q8651L8M --paths "/release/TOOLS/sts4-language-server-integrations/2023-07-31/*" + echo "List invalidations" aws cloudfront list-invalidations --distribution-id ECAO9Q8651L8M - ${{ github.workspace }}/.github/scripts/clear-s3-caches.sh ${{ inputs.path }} +# ${{ github.workspace }}/.github/scripts/clear-s3-caches.sh ${{ inputs.path }} From 180a417e32a98824914810bd7f34ef0c366a7378 Mon Sep 17 00:00:00 2001 From: aboyko Date: Mon, 31 Jul 2023 16:31:16 -0400 Subject: [PATCH 20/51] [GHA-Build] debug clear cache script --- .github/workflows/clear_s3_cache.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/clear_s3_cache.yml b/.github/workflows/clear_s3_cache.yml index f1560682c..1a52af9a5 100644 --- a/.github/workflows/clear_s3_cache.yml +++ b/.github/workflows/clear_s3_cache.yml @@ -23,6 +23,7 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.TOOLS_S3_ACCESS_KEY }} AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLS_S3_SECRET_KEY }} run: | + aws s3 cp s3://dist.springsource.com/release/TOOLS/sts4-language-server-integrations/2023-07-31 . --recursive --include "*" --dryrun echo "Firing off invalidation request" aws cloudfront create-invalidation --distribution-id ECAO9Q8651L8M --paths "/release/TOOLS/sts4-language-server-integrations/2023-07-31/*" echo "List invalidations" From 1e34337962c9a5f086c62327ecbbc9ec58c3aeec Mon Sep 17 00:00:00 2001 From: aboyko Date: Mon, 31 Jul 2023 16:36:33 -0400 Subject: [PATCH 21/51] [GHA-Build] debug clear cache script --- .github/scripts/clear-s3-caches.sh | 2 ++ .github/workflows/clear_s3_cache.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/scripts/clear-s3-caches.sh b/.github/scripts/clear-s3-caches.sh index f42710fa3..50a9aa558 100755 --- a/.github/scripts/clear-s3-caches.sh +++ b/.github/scripts/clear-s3-caches.sh @@ -1,3 +1,5 @@ +set -e + s3_path=$1 aws cloudfront create-invalidation --distribution-id ECAO9Q8651L8M --paths /release/TOOLS/sts4-language-server-integrations/2023-07-31/* invalidation_json=`aws cloudfront create-invalidation --distribution-id ECAO9Q8651L8M --output json --paths /${s3_path}/*` diff --git a/.github/workflows/clear_s3_cache.yml b/.github/workflows/clear_s3_cache.yml index 1a52af9a5..5a2010f07 100644 --- a/.github/workflows/clear_s3_cache.yml +++ b/.github/workflows/clear_s3_cache.yml @@ -22,7 +22,9 @@ jobs: env: AWS_ACCESS_KEY_ID: ${{ secrets.TOOLS_S3_ACCESS_KEY }} AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLS_S3_SECRET_KEY }} + AWS_DEFAULT_REGION: us-east-1 run: | + aws s3 cp s3://dist.springsource.com/snapshot/STS4/nightly/dist/e4.28 . --recursive --include "*" --dryrun aws s3 cp s3://dist.springsource.com/release/TOOLS/sts4-language-server-integrations/2023-07-31 . --recursive --include "*" --dryrun echo "Firing off invalidation request" aws cloudfront create-invalidation --distribution-id ECAO9Q8651L8M --paths "/release/TOOLS/sts4-language-server-integrations/2023-07-31/*" From 77d628d74e3ae81c7929a0e70240026e7b7fd737 Mon Sep 17 00:00:00 2001 From: aboyko Date: Mon, 31 Jul 2023 17:08:18 -0400 Subject: [PATCH 22/51] [GHA-Build] debug clearing cloudflare cache --- .github/scripts/clear-s3-caches.sh | 44 +++++++++++++++++++++------- .github/workflows/clear_s3_cache.yml | 16 +++++----- 2 files changed, 42 insertions(+), 18 deletions(-) diff --git a/.github/scripts/clear-s3-caches.sh b/.github/scripts/clear-s3-caches.sh index 50a9aa558..48a1637e0 100755 --- a/.github/scripts/clear-s3-caches.sh +++ b/.github/scripts/clear-s3-caches.sh @@ -1,16 +1,38 @@ set -e s3_path=$1 -aws cloudfront create-invalidation --distribution-id ECAO9Q8651L8M --paths /release/TOOLS/sts4-language-server-integrations/2023-07-31/* -invalidation_json=`aws cloudfront create-invalidation --distribution-id ECAO9Q8651L8M --output json --paths /${s3_path}/*` -echo "Invalidation response: ${invalidation_json}" -invalidation_id=`echo $invalidation_json | jq -r '.Invalidation.Id'` -invalidation_status=`echo $invalidation_json | jq -r '.Invalidation.Status'` -echo "ID=${invalidation_id} Status=${invalidation_status}" -while [ "${invalidation_status}" == "InProgress" ] +#invalidation_json=`aws cloudfront create-invalidation --distribution-id ECAO9Q8651L8M --output json --paths "/${s3_path}/*"` +#echo "Invalidation response: ${invalidation_json}" +#invalidation_id=`echo $invalidation_json | jq -r '.Invalidation.Id'` +#invalidation_status=`echo $invalidation_json | jq -r '.Invalidation.Status'` +#echo "ID=${invalidation_id} Status=${invalidation_status}" +#while [ "${invalidation_status}" == "InProgress" ] +#do +# echo "Invalidation status: ${invalidation_status}" +# sleep 3 +# invalidation_status=`aws cloudfront get-invalidation --distribution-id ECAO9Q8651L8M --id $invalidation_id --output json | jq -r '.Invalidation.Status'` +#done +#echo "Final invalidation status: ${invalidation_status}" + +s3_url=s3://dist.springsource.com/${s3_path} +files=`aws s3 cp ${s3_url} . --recursive --include "*" --dryrun` +counter=0 +json="" +for file in $files do - echo "Invalidation status: ${invalidation_status}" - sleep 3 - invalidation_status=`aws cloudfront get-invalidation --distribution-id ECAO9Q8651L8M --id $invalidation_id --output json | jq -r '.Invalidation.Status'` + if [ $counter == 0]; then + json="\"{ \"files\": [\n" + fi + if [[ "$file" =~ ^"s3://dist.springsource.com" ]]; then + let "counter++" + path=${file:26} + json="${json}\"http://dist.springsource.com${path}\",\n\"http://dist.springsource.com${path}\",\n\"http://download.springsource.com${path}\",\n\"https://download.springsource.com${path}\",\n" + fi + if [ $counter == 10]; then + json="${json:-2}\n]}" + echo $json + json="" + fi done -echo "Final invalidation status: ${invalidation_status}" + + diff --git a/.github/workflows/clear_s3_cache.yml b/.github/workflows/clear_s3_cache.yml index 5a2010f07..2cdfb2b2a 100644 --- a/.github/workflows/clear_s3_cache.yml +++ b/.github/workflows/clear_s3_cache.yml @@ -23,11 +23,13 @@ jobs: AWS_ACCESS_KEY_ID: ${{ secrets.TOOLS_S3_ACCESS_KEY }} AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLS_S3_SECRET_KEY }} AWS_DEFAULT_REGION: us-east-1 + CLOUDFLARE_ZONE_ID: ${{ secrets.CLOUDFLARE_ZONE_ID }} + CLOUDFLARE_CACHE_TOKEN: ${{ secrets.CLOUDFLARE_CACHE_TOKEN }} run: | - aws s3 cp s3://dist.springsource.com/snapshot/STS4/nightly/dist/e4.28 . --recursive --include "*" --dryrun - aws s3 cp s3://dist.springsource.com/release/TOOLS/sts4-language-server-integrations/2023-07-31 . --recursive --include "*" --dryrun - echo "Firing off invalidation request" - aws cloudfront create-invalidation --distribution-id ECAO9Q8651L8M --paths "/release/TOOLS/sts4-language-server-integrations/2023-07-31/*" - echo "List invalidations" - aws cloudfront list-invalidations --distribution-id ECAO9Q8651L8M -# ${{ github.workspace }}/.github/scripts/clear-s3-caches.sh ${{ inputs.path }} +# aws s3 cp s3://dist.springsource.com/snapshot/STS4/nightly/dist/e4.28 . --recursive --include "*" --dryrun +# aws s3 cp s3://dist.springsource.com/release/TOOLS/sts4-language-server-integrations/2023-07-31 . --recursive --include "*" --dryrun +# echo "Firing off invalidation request" +# aws cloudfront create-invalidation --distribution-id ECAO9Q8651L8M --paths "/release/TOOLS/sts4-language-server-integrations/2023-07-31/*" +# echo "List invalidations" +# aws cloudfront list-invalidations --distribution-id ECAO9Q8651L8M + ${{ github.workspace }}/.github/scripts/clear-s3-caches.sh ${{ inputs.path }} From dfa21c635543d1a1efdf2a5f00c8ad4f08abb42a Mon Sep 17 00:00:00 2001 From: aboyko Date: Mon, 31 Jul 2023 17:11:12 -0400 Subject: [PATCH 23/51] [GHA-Build] fix syntax error --- .github/workflows/clear_s3_cache.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/.github/workflows/clear_s3_cache.yml b/.github/workflows/clear_s3_cache.yml index 2cdfb2b2a..bf937454b 100644 --- a/.github/workflows/clear_s3_cache.yml +++ b/.github/workflows/clear_s3_cache.yml @@ -26,10 +26,4 @@ jobs: CLOUDFLARE_ZONE_ID: ${{ secrets.CLOUDFLARE_ZONE_ID }} CLOUDFLARE_CACHE_TOKEN: ${{ secrets.CLOUDFLARE_CACHE_TOKEN }} run: | -# aws s3 cp s3://dist.springsource.com/snapshot/STS4/nightly/dist/e4.28 . --recursive --include "*" --dryrun -# aws s3 cp s3://dist.springsource.com/release/TOOLS/sts4-language-server-integrations/2023-07-31 . --recursive --include "*" --dryrun -# echo "Firing off invalidation request" -# aws cloudfront create-invalidation --distribution-id ECAO9Q8651L8M --paths "/release/TOOLS/sts4-language-server-integrations/2023-07-31/*" -# echo "List invalidations" -# aws cloudfront list-invalidations --distribution-id ECAO9Q8651L8M ${{ github.workspace }}/.github/scripts/clear-s3-caches.sh ${{ inputs.path }} From 21f49d95295a7b95df3dc5443b2a3f68129f838d Mon Sep 17 00:00:00 2001 From: aboyko Date: Mon, 31 Jul 2023 17:15:20 -0400 Subject: [PATCH 24/51] [GHA-Build] integer comparisons --- .github/scripts/clear-s3-caches.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/scripts/clear-s3-caches.sh b/.github/scripts/clear-s3-caches.sh index 48a1637e0..94e4a823b 100755 --- a/.github/scripts/clear-s3-caches.sh +++ b/.github/scripts/clear-s3-caches.sh @@ -20,15 +20,15 @@ counter=0 json="" for file in $files do - if [ $counter == 0]; then + if [[ "$counter" eq 0 ]]; then json="\"{ \"files\": [\n" fi if [[ "$file" =~ ^"s3://dist.springsource.com" ]]; then - let "counter++" + ((counter++)) path=${file:26} json="${json}\"http://dist.springsource.com${path}\",\n\"http://dist.springsource.com${path}\",\n\"http://download.springsource.com${path}\",\n\"https://download.springsource.com${path}\",\n" fi - if [ $counter == 10]; then + if [[ "$counter" eq 10 ]]; then json="${json:-2}\n]}" echo $json json="" From c49577fbaaca9b1677e8ea5956b4422561f625cf Mon Sep 17 00:00:00 2001 From: aboyko Date: Mon, 31 Jul 2023 17:17:14 -0400 Subject: [PATCH 25/51] [GHA-Build] integer comparisons with -eq --- .github/scripts/clear-s3-caches.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/scripts/clear-s3-caches.sh b/.github/scripts/clear-s3-caches.sh index 94e4a823b..4603961aa 100755 --- a/.github/scripts/clear-s3-caches.sh +++ b/.github/scripts/clear-s3-caches.sh @@ -20,7 +20,7 @@ counter=0 json="" for file in $files do - if [[ "$counter" eq 0 ]]; then + if [[ "$counter" -eq 0 ]]; then json="\"{ \"files\": [\n" fi if [[ "$file" =~ ^"s3://dist.springsource.com" ]]; then @@ -28,7 +28,7 @@ do path=${file:26} json="${json}\"http://dist.springsource.com${path}\",\n\"http://dist.springsource.com${path}\",\n\"http://download.springsource.com${path}\",\n\"https://download.springsource.com${path}\",\n" fi - if [[ "$counter" eq 10 ]]; then + if [[ "$counter" -eq 10 ]]; then json="${json:-2}\n]}" echo $json json="" From 427783e2a7110d8e906cb5013efdad4e6cd508d5 Mon Sep 17 00:00:00 2001 From: aboyko Date: Mon, 31 Jul 2023 17:21:34 -0400 Subject: [PATCH 26/51] [GHA-Build] Debug cloudflare cache flush script --- .github/scripts/clear-s3-caches.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/scripts/clear-s3-caches.sh b/.github/scripts/clear-s3-caches.sh index 4603961aa..c2f70e994 100755 --- a/.github/scripts/clear-s3-caches.sh +++ b/.github/scripts/clear-s3-caches.sh @@ -20,15 +20,21 @@ counter=0 json="" for file in $files do + echo "Processing ${file}" if [[ "$counter" -eq 0 ]]; then json="\"{ \"files\": [\n" fi + echo "Current json: ${json}" if [[ "$file" =~ ^"s3://dist.springsource.com" ]]; then + echo "Is a file in s3 ${file}" ((counter++)) + echo "Counter ${counter}" path=${file:26} + echo "Path ${path}" json="${json}\"http://dist.springsource.com${path}\",\n\"http://dist.springsource.com${path}\",\n\"http://download.springsource.com${path}\",\n\"https://download.springsource.com${path}\",\n" fi if [[ "$counter" -eq 10 ]]; then + echo "Batch completed" json="${json:-2}\n]}" echo $json json="" From 4068332485f6711e27407a0e76cf12b92bf92708 Mon Sep 17 00:00:00 2001 From: aboyko Date: Mon, 31 Jul 2023 17:28:13 -0400 Subject: [PATCH 27/51] [GHA-Build] Debug cloudflare cache flush script --- .github/scripts/clear-s3-caches.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/scripts/clear-s3-caches.sh b/.github/scripts/clear-s3-caches.sh index c2f70e994..9c44a72b2 100755 --- a/.github/scripts/clear-s3-caches.sh +++ b/.github/scripts/clear-s3-caches.sh @@ -22,20 +22,20 @@ for file in $files do echo "Processing ${file}" if [[ "$counter" -eq 0 ]]; then - json="\"{ \"files\": [\n" + json="{\"files\": [$'\n'" fi echo "Current json: ${json}" if [[ "$file" =~ ^"s3://dist.springsource.com" ]]; then echo "Is a file in s3 ${file}" - ((counter++)) + let "counter++" echo "Counter ${counter}" path=${file:26} echo "Path ${path}" - json="${json}\"http://dist.springsource.com${path}\",\n\"http://dist.springsource.com${path}\",\n\"http://download.springsource.com${path}\",\n\"https://download.springsource.com${path}\",\n" + json="${json}\"http://dist.springsource.com${path}\",$'\n'\"http://dist.springsource.com${path}\",$'\n'\"http://download.springsource.com${path}\",$'\n'\"https://download.springsource.com${path}\",$'\n'" fi if [[ "$counter" -eq 10 ]]; then echo "Batch completed" - json="${json:-2}\n]}" + json="${json:-2}$'\n']}" echo $json json="" fi From 180b2c139e39dfc7c562d8bfedd04200b53696b0 Mon Sep 17 00:00:00 2001 From: aboyko Date: Mon, 31 Jul 2023 17:33:32 -0400 Subject: [PATCH 28/51] [GHA-Build] Debug cloudflare cache flush script --- .github/scripts/clear-s3-caches.sh | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/scripts/clear-s3-caches.sh b/.github/scripts/clear-s3-caches.sh index 9c44a72b2..4051526d8 100755 --- a/.github/scripts/clear-s3-caches.sh +++ b/.github/scripts/clear-s3-caches.sh @@ -18,24 +18,25 @@ s3_url=s3://dist.springsource.com/${s3_path} files=`aws s3 cp ${s3_url} . --recursive --include "*" --dryrun` counter=0 json="" +NL=$'\n' for file in $files do echo "Processing ${file}" if [[ "$counter" -eq 0 ]]; then - json="{\"files\": [$'\n'" + json="{\"files\": [${NL}" fi echo "Current json: ${json}" if [[ "$file" =~ ^"s3://dist.springsource.com" ]]; then echo "Is a file in s3 ${file}" - let "counter++" + counter=$((counter+1)) echo "Counter ${counter}" path=${file:26} echo "Path ${path}" - json="${json}\"http://dist.springsource.com${path}\",$'\n'\"http://dist.springsource.com${path}\",$'\n'\"http://download.springsource.com${path}\",$'\n'\"https://download.springsource.com${path}\",$'\n'" + json="${json}\"http://dist.springsource.com${path}\",${NL}\"http://dist.springsource.com${path}\",${NL}\"http://download.springsource.com${path}\",${NL}\"https://download.springsource.com${path}\",${NL}" fi if [[ "$counter" -eq 10 ]]; then echo "Batch completed" - json="${json:-2}$'\n']}" + json="${json:-2}${NL}]}" echo $json json="" fi From d60a426048dc2f9b62bf540c2b7bdc23288a8be6 Mon Sep 17 00:00:00 2001 From: aboyko Date: Mon, 31 Jul 2023 17:35:32 -0400 Subject: [PATCH 29/51] [GHA-Build] Debug cloudflare cache flush script --- .github/scripts/clear-s3-caches.sh | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/.github/scripts/clear-s3-caches.sh b/.github/scripts/clear-s3-caches.sh index 4051526d8..aba59a553 100755 --- a/.github/scripts/clear-s3-caches.sh +++ b/.github/scripts/clear-s3-caches.sh @@ -25,20 +25,16 @@ do if [[ "$counter" -eq 0 ]]; then json="{\"files\": [${NL}" fi - echo "Current json: ${json}" if [[ "$file" =~ ^"s3://dist.springsource.com" ]]; then - echo "Is a file in s3 ${file}" counter=$((counter+1)) - echo "Counter ${counter}" path=${file:26} - echo "Path ${path}" json="${json}\"http://dist.springsource.com${path}\",${NL}\"http://dist.springsource.com${path}\",${NL}\"http://download.springsource.com${path}\",${NL}\"https://download.springsource.com${path}\",${NL}" fi if [[ "$counter" -eq 10 ]]; then - echo "Batch completed" json="${json:-2}${NL}]}" echo $json json="" + counter=0 fi done From a5f19fb927f3dcdfcaf521571d8676278124e11e Mon Sep 17 00:00:00 2001 From: aboyko Date: Mon, 31 Jul 2023 17:41:06 -0400 Subject: [PATCH 30/51] [GHA-Build] Debug cloudflare cache flush script --- .github/scripts/clear-s3-caches.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/scripts/clear-s3-caches.sh b/.github/scripts/clear-s3-caches.sh index aba59a553..6f10f0de6 100755 --- a/.github/scripts/clear-s3-caches.sh +++ b/.github/scripts/clear-s3-caches.sh @@ -21,21 +21,26 @@ json="" NL=$'\n' for file in $files do - echo "Processing ${file}" if [[ "$counter" -eq 0 ]]; then json="{\"files\": [${NL}" fi if [[ "$file" =~ ^"s3://dist.springsource.com" ]]; then counter=$((counter+1)) path=${file:26} - json="${json}\"http://dist.springsource.com${path}\",${NL}\"http://dist.springsource.com${path}\",${NL}\"http://download.springsource.com${path}\",${NL}\"https://download.springsource.com${path}\",${NL}" + json="${json}\"http://dist.springsource.com${path}\",${NL}\"https://dist.springsource.com${path}\",${NL}\"http://download.springsource.com${path}\",${NL}\"https://download.springsource.com${path}\",${NL}" fi if [[ "$counter" -eq 10 ]]; then json="${json:-2}${NL}]}" echo $json json="" counter=0 + # flush with request fi done +if ! [[ "$counter" -eq 0 ]]; then + json="${json:-2}${NL}]}" + echo $json + # flush with request +fi From be38bb06b04139f1073bdc4b252c7088098d79cc Mon Sep 17 00:00:00 2001 From: aboyko Date: Mon, 31 Jul 2023 17:41:51 -0400 Subject: [PATCH 31/51] [GHA-Build] Debug cloudflare cache flush script --- .github/scripts/clear-s3-caches.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/scripts/clear-s3-caches.sh b/.github/scripts/clear-s3-caches.sh index 6f10f0de6..5a204c706 100755 --- a/.github/scripts/clear-s3-caches.sh +++ b/.github/scripts/clear-s3-caches.sh @@ -25,6 +25,7 @@ do json="{\"files\": [${NL}" fi if [[ "$file" =~ ^"s3://dist.springsource.com" ]]; then + echo "Processing ${file}" counter=$((counter+1)) path=${file:26} json="${json}\"http://dist.springsource.com${path}\",${NL}\"https://dist.springsource.com${path}\",${NL}\"http://download.springsource.com${path}\",${NL}\"https://download.springsource.com${path}\",${NL}" From 4cdba7ca432abcfac0219e0db5da9f9323d024ce Mon Sep 17 00:00:00 2001 From: aboyko Date: Mon, 31 Jul 2023 17:45:02 -0400 Subject: [PATCH 32/51] [GHA-Build] Debug cloudflare cache flush script --- .github/scripts/clear-s3-caches.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/scripts/clear-s3-caches.sh b/.github/scripts/clear-s3-caches.sh index 5a204c706..22c39bf93 100755 --- a/.github/scripts/clear-s3-caches.sh +++ b/.github/scripts/clear-s3-caches.sh @@ -31,7 +31,7 @@ do json="${json}\"http://dist.springsource.com${path}\",${NL}\"https://dist.springsource.com${path}\",${NL}\"http://download.springsource.com${path}\",${NL}\"https://download.springsource.com${path}\",${NL}" fi if [[ "$counter" -eq 10 ]]; then - json="${json:-2}${NL}]}" + json="${json::-2}${NL}]}" echo $json json="" counter=0 @@ -39,7 +39,7 @@ do fi done if ! [[ "$counter" -eq 0 ]]; then - json="${json:-2}${NL}]}" + json="${json::-2}${NL}]}" echo $json # flush with request fi From 8c4bc8a3ef0ffe398163f15c83cf51294959b4cc Mon Sep 17 00:00:00 2001 From: aboyko Date: Mon, 31 Jul 2023 17:55:01 -0400 Subject: [PATCH 33/51] [GHA-Build] Try live clearing cloudflare cache --- .github/scripts/clear-s3-caches.sh | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/scripts/clear-s3-caches.sh b/.github/scripts/clear-s3-caches.sh index 22c39bf93..0e678b710 100755 --- a/.github/scripts/clear-s3-caches.sh +++ b/.github/scripts/clear-s3-caches.sh @@ -14,6 +14,7 @@ s3_path=$1 #done #echo "Final invalidation status: ${invalidation_status}" +# Flush CloudFlare cache s3_url=s3://dist.springsource.com/${s3_path} files=`aws s3 cp ${s3_url} . --recursive --include "*" --dryrun` counter=0 @@ -33,15 +34,26 @@ do if [[ "$counter" -eq 10 ]]; then json="${json::-2}${NL}]}" echo $json + + curl -X DELETE "https://api.cloudflare.com/client/v4/zones/${CLOUDFLARE_ZONE_ID}/purge_cache" \ + -H "X-Auth-Email: spring-sysadmin@pivotal.io" \ + -H "Authorization: Bearer ${CLOUDFLARE_CACHE_TOKEN}" \ + -H "Content-Type: application/json" \ + --data "${json}" + json="" counter=0 - # flush with request fi done if ! [[ "$counter" -eq 0 ]]; then json="${json::-2}${NL}]}" echo $json - # flush with request + + curl -X DELETE "https://api.cloudflare.com/client/v4/zones/${CLOUDFLARE_ZONE_ID}/purge_cache" \ + -H "X-Auth-Email: spring-sysadmin@pivotal.io" \ + -H "Authorization: Bearer ${CLOUDFLARE_CACHE_TOKEN}" \ + -H "Content-Type: application/json" \ + --data "${json}" fi From 1dd417ae4bee2151ccc23c3add0949af37279ebf Mon Sep 17 00:00:00 2001 From: aboyko Date: Mon, 31 Jul 2023 18:00:19 -0400 Subject: [PATCH 34/51] [GHA-Build] Decrement batch size for cloudflare cache clearance --- .github/scripts/clear-s3-caches.sh | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/.github/scripts/clear-s3-caches.sh b/.github/scripts/clear-s3-caches.sh index 0e678b710..c0d5a553b 100755 --- a/.github/scripts/clear-s3-caches.sh +++ b/.github/scripts/clear-s3-caches.sh @@ -20,6 +20,7 @@ files=`aws s3 cp ${s3_url} . --recursive --include "*" --dryrun` counter=0 json="" NL=$'\n' +FILES_BATCH=7 for file in $files do if [[ "$counter" -eq 0 ]]; then @@ -31,15 +32,15 @@ do path=${file:26} json="${json}\"http://dist.springsource.com${path}\",${NL}\"https://dist.springsource.com${path}\",${NL}\"http://download.springsource.com${path}\",${NL}\"https://download.springsource.com${path}\",${NL}" fi - if [[ "$counter" -eq 10 ]]; then + if [[ "$counter" -eq "$FILES_BATCH" ]]; then json="${json::-2}${NL}]}" echo $json - curl -X DELETE "https://api.cloudflare.com/client/v4/zones/${CLOUDFLARE_ZONE_ID}/purge_cache" \ - -H "X-Auth-Email: spring-sysadmin@pivotal.io" \ - -H "Authorization: Bearer ${CLOUDFLARE_CACHE_TOKEN}" \ - -H "Content-Type: application/json" \ - --data "${json}" +# curl -X DELETE "https://api.cloudflare.com/client/v4/zones/${CLOUDFLARE_ZONE_ID}/purge_cache" \ +# -H "X-Auth-Email: spring-sysadmin@pivotal.io" \ +# -H "Authorization: Bearer ${CLOUDFLARE_CACHE_TOKEN}" \ +# -H "Content-Type: application/json" \ +# --data "${json}" json="" counter=0 @@ -49,11 +50,11 @@ if ! [[ "$counter" -eq 0 ]]; then json="${json::-2}${NL}]}" echo $json - curl -X DELETE "https://api.cloudflare.com/client/v4/zones/${CLOUDFLARE_ZONE_ID}/purge_cache" \ - -H "X-Auth-Email: spring-sysadmin@pivotal.io" \ - -H "Authorization: Bearer ${CLOUDFLARE_CACHE_TOKEN}" \ - -H "Content-Type: application/json" \ - --data "${json}" +# curl -X DELETE "https://api.cloudflare.com/client/v4/zones/${CLOUDFLARE_ZONE_ID}/purge_cache" \ +# -H "X-Auth-Email: spring-sysadmin@pivotal.io" \ +# -H "Authorization: Bearer ${CLOUDFLARE_CACHE_TOKEN}" \ +# -H "Content-Type: application/json" \ +# --data "${json}" fi From 103ae586b5e58cb73c65b505a0126e9f8a76aa4a Mon Sep 17 00:00:00 2001 From: aboyko Date: Mon, 31 Jul 2023 18:02:06 -0400 Subject: [PATCH 35/51] [GHA-Build] Another attempt at live cloudflare cache clearance --- .github/scripts/clear-s3-caches.sh | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/scripts/clear-s3-caches.sh b/.github/scripts/clear-s3-caches.sh index c0d5a553b..761c2f804 100755 --- a/.github/scripts/clear-s3-caches.sh +++ b/.github/scripts/clear-s3-caches.sh @@ -36,11 +36,11 @@ do json="${json::-2}${NL}]}" echo $json -# curl -X DELETE "https://api.cloudflare.com/client/v4/zones/${CLOUDFLARE_ZONE_ID}/purge_cache" \ -# -H "X-Auth-Email: spring-sysadmin@pivotal.io" \ -# -H "Authorization: Bearer ${CLOUDFLARE_CACHE_TOKEN}" \ -# -H "Content-Type: application/json" \ -# --data "${json}" + curl -X DELETE "https://api.cloudflare.com/client/v4/zones/${CLOUDFLARE_ZONE_ID}/purge_cache" \ + -H "X-Auth-Email: spring-sysadmin@pivotal.io" \ + -H "Authorization: Bearer ${CLOUDFLARE_CACHE_TOKEN}" \ + -H "Content-Type: application/json" \ + --data "${json}" json="" counter=0 @@ -50,11 +50,11 @@ if ! [[ "$counter" -eq 0 ]]; then json="${json::-2}${NL}]}" echo $json -# curl -X DELETE "https://api.cloudflare.com/client/v4/zones/${CLOUDFLARE_ZONE_ID}/purge_cache" \ -# -H "X-Auth-Email: spring-sysadmin@pivotal.io" \ -# -H "Authorization: Bearer ${CLOUDFLARE_CACHE_TOKEN}" \ -# -H "Content-Type: application/json" \ -# --data "${json}" + curl -X DELETE "https://api.cloudflare.com/client/v4/zones/${CLOUDFLARE_ZONE_ID}/purge_cache" \ + -H "X-Auth-Email: spring-sysadmin@pivotal.io" \ + -H "Authorization: Bearer ${CLOUDFLARE_CACHE_TOKEN}" \ + -H "Content-Type: application/json" \ + --data "${json}" fi From 96947665c16f16e9a9bf07c222b573723d48b668 Mon Sep 17 00:00:00 2001 From: aboyko Date: Mon, 31 Jul 2023 18:32:04 -0400 Subject: [PATCH 36/51] [GHA-Build] Store s3 dist folders in files to use by GHA --- .github/workflows/eclipse-ls-extensions-build.yml | 11 +++++++++++ .../pom.xml | 3 +++ .../pom.xml | 3 +++ .../pom.xml | 3 +++ .../pom.xml | 14 ++++++++++++++ 5 files changed, 34 insertions(+) diff --git a/.github/workflows/eclipse-ls-extensions-build.yml b/.github/workflows/eclipse-ls-extensions-build.yml index 9ad3b71c6..a88543cb9 100644 --- a/.github/workflows/eclipse-ls-extensions-build.yml +++ b/.github/workflows/eclipse-ls-extensions-build.yml @@ -78,6 +78,17 @@ jobs: else ./mvnw --batch-mode clean deploy -P${{ inputs.eclipse_profile }} -P${{ inputs.build_type }} $sts4_ls_version_param -Pgitactions -Pgpg.sign -Dmaven.test.skip=true -Dsigning.skip=true -Dhttpclient.retry-max=20 -Declipse.p2.mirrors=false -Ds3service.https-only=true -Dtycho.localArtifacts=ignore fi + - name: + if: ${{ inputs.build_type !== 'snapshot' }} + env: + AWS_ACCESS_KEY_ID: ${{ secrets.TOOLS_S3_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLS_S3_SECRET_KEY }} + AWS_DEFAULT_REGION: us-east-1 + CLOUDFLARE_ZONE_ID: ${{ secrets.CLOUDFLARE_ZONE_ID }} + CLOUDFLARE_CACHE_TOKEN: ${{ secrets.CLOUDFLARE_CACHE_TOKEN }} + run: | + dist_path=`cat ${{ github.workspace }}/eclipse-language-servers/org.springframework.tooling.ls.integration.repository/target/s3-p2-repo-dist-path.txt` + echo $dist_path - name: Verify Eclipse LS Extensions on e429 run: | cd eclipse-language-servers diff --git a/eclipse-distribution/org.springframework.boot.ide.product.e427/pom.xml b/eclipse-distribution/org.springframework.boot.ide.product.e427/pom.xml index 92a5195e2..12cc7f0a9 100644 --- a/eclipse-distribution/org.springframework.boot.ide.product.e427/pom.xml +++ b/eclipse-distribution/org.springframework.boot.ide.product.e427/pom.xml @@ -387,6 +387,9 @@ ${dist.path.product} + + ${dist.path.repo} + diff --git a/eclipse-distribution/org.springframework.boot.ide.product.e428/pom.xml b/eclipse-distribution/org.springframework.boot.ide.product.e428/pom.xml index d02a5b0ee..d9f1ac3c3 100644 --- a/eclipse-distribution/org.springframework.boot.ide.product.e428/pom.xml +++ b/eclipse-distribution/org.springframework.boot.ide.product.e428/pom.xml @@ -375,6 +375,9 @@ ${dist.path.product} + + ${dist.path.repo} + diff --git a/eclipse-distribution/org.springframework.boot.ide.product.e429/pom.xml b/eclipse-distribution/org.springframework.boot.ide.product.e429/pom.xml index 6b9b0902a..e39d98f81 100644 --- a/eclipse-distribution/org.springframework.boot.ide.product.e429/pom.xml +++ b/eclipse-distribution/org.springframework.boot.ide.product.e429/pom.xml @@ -388,6 +388,9 @@ ${dist.path.product} + + ${dist.path.repo} + diff --git a/eclipse-language-servers/org.springframework.tooling.ls.integration.repository/pom.xml b/eclipse-language-servers/org.springframework.tooling.ls.integration.repository/pom.xml index 211488f5c..4c13ce83b 100644 --- a/eclipse-language-servers/org.springframework.tooling.ls.integration.repository/pom.xml +++ b/eclipse-language-servers/org.springframework.tooling.ls.integration.repository/pom.xml @@ -35,6 +35,20 @@ maven-antrun-plugin 1.6 + + generate-s3-upload-info + verify + + + + ${dist.path} + + + + + run + + upload-repo deploy From f09bf3ad9868b6bc35e14db7df1940cc656a868f Mon Sep 17 00:00:00 2001 From: aboyko Date: Mon, 31 Jul 2023 18:34:14 -0400 Subject: [PATCH 37/51] [GHA-Build] Fix yml syntax error --- .github/workflows/eclipse-ls-extensions-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/eclipse-ls-extensions-build.yml b/.github/workflows/eclipse-ls-extensions-build.yml index a88543cb9..e04d370af 100644 --- a/.github/workflows/eclipse-ls-extensions-build.yml +++ b/.github/workflows/eclipse-ls-extensions-build.yml @@ -79,7 +79,7 @@ jobs: ./mvnw --batch-mode clean deploy -P${{ inputs.eclipse_profile }} -P${{ inputs.build_type }} $sts4_ls_version_param -Pgitactions -Pgpg.sign -Dmaven.test.skip=true -Dsigning.skip=true -Dhttpclient.retry-max=20 -Declipse.p2.mirrors=false -Ds3service.https-only=true -Dtycho.localArtifacts=ignore fi - name: - if: ${{ inputs.build_type !== 'snapshot' }} + if: ${{ inputs.build_type != 'snapshot' }} env: AWS_ACCESS_KEY_ID: ${{ secrets.TOOLS_S3_ACCESS_KEY }} AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLS_S3_SECRET_KEY }} From 0caec5b6580ef9ada8598beb511c675bc6ea4154 Mon Sep 17 00:00:00 2001 From: aboyko Date: Mon, 31 Jul 2023 19:00:59 -0400 Subject: [PATCH 38/51] [GHA-Build] Placeholders for clearing S3 caches in the builds --- .github/scripts/clear-s3-caches.sh | 4 ++- .../workflows/eclipse-ls-extensions-build.yml | 4 ++- .../gh-hosted-eclipse-distro-build.yml | 29 +++++++++++++++++++ 3 files changed, 35 insertions(+), 2 deletions(-) diff --git a/.github/scripts/clear-s3-caches.sh b/.github/scripts/clear-s3-caches.sh index 761c2f804..977f2acfd 100755 --- a/.github/scripts/clear-s3-caches.sh +++ b/.github/scripts/clear-s3-caches.sh @@ -1,6 +1,8 @@ set -e s3_path=$1 + +#Flush AWS Cloudfront Cache #invalidation_json=`aws cloudfront create-invalidation --distribution-id ECAO9Q8651L8M --output json --paths "/${s3_path}/*"` #echo "Invalidation response: ${invalidation_json}" #invalidation_id=`echo $invalidation_json | jq -r '.Invalidation.Id'` @@ -14,7 +16,7 @@ s3_path=$1 #done #echo "Final invalidation status: ${invalidation_status}" -# Flush CloudFlare cache +# Flush CloudFlare Cache s3_url=s3://dist.springsource.com/${s3_path} files=`aws s3 cp ${s3_url} . --recursive --include "*" --dryrun` counter=0 diff --git a/.github/workflows/eclipse-ls-extensions-build.yml b/.github/workflows/eclipse-ls-extensions-build.yml index e04d370af..3d8400530 100644 --- a/.github/workflows/eclipse-ls-extensions-build.yml +++ b/.github/workflows/eclipse-ls-extensions-build.yml @@ -39,6 +39,7 @@ jobs: - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 with: sparse-checkout: | + .github headless-services eclipse-language-servers - name: Set up JDK 17 @@ -78,7 +79,7 @@ jobs: else ./mvnw --batch-mode clean deploy -P${{ inputs.eclipse_profile }} -P${{ inputs.build_type }} $sts4_ls_version_param -Pgitactions -Pgpg.sign -Dmaven.test.skip=true -Dsigning.skip=true -Dhttpclient.retry-max=20 -Declipse.p2.mirrors=false -Ds3service.https-only=true -Dtycho.localArtifacts=ignore fi - - name: + - name: Clear S3 Caches if: ${{ inputs.build_type != 'snapshot' }} env: AWS_ACCESS_KEY_ID: ${{ secrets.TOOLS_S3_ACCESS_KEY }} @@ -90,6 +91,7 @@ jobs: dist_path=`cat ${{ github.workspace }}/eclipse-language-servers/org.springframework.tooling.ls.integration.repository/target/s3-p2-repo-dist-path.txt` echo $dist_path - name: Verify Eclipse LS Extensions on e429 + if: ${{ inputs.build_type == 'snapshot' }} run: | cd eclipse-language-servers if command -v xvfb-run ; then diff --git a/.github/workflows/gh-hosted-eclipse-distro-build.yml b/.github/workflows/gh-hosted-eclipse-distro-build.yml index c533efffc..18a83e06d 100644 --- a/.github/workflows/gh-hosted-eclipse-distro-build.yml +++ b/.github/workflows/gh-hosted-eclipse-distro-build.yml @@ -90,6 +90,17 @@ jobs: aws s3 rm s3://dist.springsource.com/${dist_path} --recursive --exclude "*" --include "spring-tool-suite-4*linux*.tar.gz*" --exclude "*/*" echo "Uploading new Linux .ta.gz files to s3..." aws s3 cp . s3://dist.springsource.com/${dist_path} --recursive --exclude "*" --include "spring-tool-suite-4*linux*.tar.gz*" --exclude "*/*" --acl public-read --no-progress + - name: Clear S3 Caches for P2 Repo + if: ${{ inputs.build_type != 'snapshot' }} + env: + AWS_ACCESS_KEY_ID: ${{ secrets.TOOLS_S3_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLS_S3_SECRET_KEY }} + AWS_DEFAULT_REGION: us-east-1 + CLOUDFLARE_ZONE_ID: ${{ secrets.CLOUDFLARE_ZONE_ID }} + CLOUDFLARE_CACHE_TOKEN: ${{ secrets.CLOUDFLARE_CACHE_TOKEN }} + run: | + dist_path=`cat ${{ github.workspace }}/eclipse-distribution/org.springframework.boot.ide.product.${{ inputs.eclipse_profile }}/target/s3-p2-repo-dist-path.txt` + echo $dist_path - name: Update Nightly Distro Downloads page if: ${{ inputs.build_type == 'snapshot' && always() }} env: @@ -280,6 +291,24 @@ jobs: if: ${{ always() }} runs-on: ubuntu-latest steps: + - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 + with: + sparse-checkout: | + .github + - uses: actions/download-artifact@cbed621e49e4c01b044d60f6c80ea4ed6328b281 + with: + name: s3-dist-path-${{ github.run_id }}.${{ github.run_attempt }} + - name: Clear S3 Caches for Distros + if: ${{ inputs.build_type != 'snapshot' }} + env: + AWS_ACCESS_KEY_ID: ${{ secrets.TOOLS_S3_ACCESS_KEY }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLS_S3_SECRET_KEY }} + AWS_DEFAULT_REGION: us-east-1 + CLOUDFLARE_ZONE_ID: ${{ secrets.CLOUDFLARE_ZONE_ID }} + CLOUDFLARE_CACHE_TOKEN: ${{ secrets.CLOUDFLARE_CACHE_TOKEN }} + run: | + dist_path=`cat s3-dist-path.txt` + echo $dist_path - name: Remove Temp Build Artifacts from S3 id: cleanup-s3-temp-storage env: From 8228a46ed530e343c5d09327d19ab22606f4ec40 Mon Sep 17 00:00:00 2001 From: aboyko Date: Mon, 31 Jul 2023 19:01:54 -0400 Subject: [PATCH 39/51] [GHA-Build] Adjust workflow name --- .github/workflows/release-eclipse-distro-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release-eclipse-distro-build.yml b/.github/workflows/release-eclipse-distro-build.yml index c9c1913c8..fe0ca5da9 100644 --- a/.github/workflows/release-eclipse-distro-build.yml +++ b/.github/workflows/release-eclipse-distro-build.yml @@ -1,4 +1,4 @@ -name: Release - Eclipse Build +name: Release - Eclipse Distros Build concurrency: group: release-eclipse From 2a4df722b7352898d573bafe4d33a3d0fe850832 Mon Sep 17 00:00:00 2001 From: aboyko Date: Mon, 31 Jul 2023 19:08:33 -0400 Subject: [PATCH 40/51] [GHA-Build] Try permissions to flush cloudfront cache --- .github/workflows/clear_s3_cache.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/clear_s3_cache.yml b/.github/workflows/clear_s3_cache.yml index bf937454b..48d4f6b11 100644 --- a/.github/workflows/clear_s3_cache.yml +++ b/.github/workflows/clear_s3_cache.yml @@ -26,4 +26,5 @@ jobs: CLOUDFLARE_ZONE_ID: ${{ secrets.CLOUDFLARE_ZONE_ID }} CLOUDFLARE_CACHE_TOKEN: ${{ secrets.CLOUDFLARE_CACHE_TOKEN }} run: | + aws cloudfront create-invalidation --distribution-id ECAO9Q8651L8M --paths '/release/TOOLS/sts4-language-server-integrations/2023-07-31/*' ${{ github.workspace }}/.github/scripts/clear-s3-caches.sh ${{ inputs.path }} From 14d0f943b5e30517b6fc6f2bd545250b28ce008b Mon Sep 17 00:00:00 2001 From: aboyko Date: Mon, 31 Jul 2023 19:23:27 -0400 Subject: [PATCH 41/51] [GHA-Build] Try cloudfront cache flush script --- .github/scripts/clear-s3-caches.sh | 106 +++++++++++++-------------- .github/workflows/clear_s3_cache.yml | 1 - 2 files changed, 53 insertions(+), 54 deletions(-) diff --git a/.github/scripts/clear-s3-caches.sh b/.github/scripts/clear-s3-caches.sh index 977f2acfd..8a665a675 100755 --- a/.github/scripts/clear-s3-caches.sh +++ b/.github/scripts/clear-s3-caches.sh @@ -3,60 +3,60 @@ set -e s3_path=$1 #Flush AWS Cloudfront Cache -#invalidation_json=`aws cloudfront create-invalidation --distribution-id ECAO9Q8651L8M --output json --paths "/${s3_path}/*"` -#echo "Invalidation response: ${invalidation_json}" -#invalidation_id=`echo $invalidation_json | jq -r '.Invalidation.Id'` -#invalidation_status=`echo $invalidation_json | jq -r '.Invalidation.Status'` -#echo "ID=${invalidation_id} Status=${invalidation_status}" -#while [ "${invalidation_status}" == "InProgress" ] -#do -# echo "Invalidation status: ${invalidation_status}" -# sleep 3 -# invalidation_status=`aws cloudfront get-invalidation --distribution-id ECAO9Q8651L8M --id $invalidation_id --output json | jq -r '.Invalidation.Status'` -#done -#echo "Final invalidation status: ${invalidation_status}" +invalidation_json=`aws cloudfront create-invalidation --distribution-id ECAO9Q8651L8M --output json --paths "/${s3_path}/*"` +echo "Invalidation response: ${invalidation_json}" +invalidation_id=`echo $invalidation_json | jq -r '.Invalidation.Id'` +invalidation_status=`echo $invalidation_json | jq -r '.Invalidation.Status'` +echo "ID=${invalidation_id} Status=${invalidation_status}" +while [ "${invalidation_status}" == "InProgress" ] +do + echo "Invalidation status: ${invalidation_status}" + sleep 3 + invalidation_status=`aws cloudfront get-invalidation --distribution-id ECAO9Q8651L8M --id $invalidation_id --output json | jq -r '.Invalidation.Status'` +done +echo "Final invalidation status: ${invalidation_status}" # Flush CloudFlare Cache -s3_url=s3://dist.springsource.com/${s3_path} -files=`aws s3 cp ${s3_url} . --recursive --include "*" --dryrun` -counter=0 -json="" -NL=$'\n' -FILES_BATCH=7 -for file in $files -do - if [[ "$counter" -eq 0 ]]; then - json="{\"files\": [${NL}" - fi - if [[ "$file" =~ ^"s3://dist.springsource.com" ]]; then - echo "Processing ${file}" - counter=$((counter+1)) - path=${file:26} - json="${json}\"http://dist.springsource.com${path}\",${NL}\"https://dist.springsource.com${path}\",${NL}\"http://download.springsource.com${path}\",${NL}\"https://download.springsource.com${path}\",${NL}" - fi - if [[ "$counter" -eq "$FILES_BATCH" ]]; then - json="${json::-2}${NL}]}" - echo $json - - curl -X DELETE "https://api.cloudflare.com/client/v4/zones/${CLOUDFLARE_ZONE_ID}/purge_cache" \ - -H "X-Auth-Email: spring-sysadmin@pivotal.io" \ - -H "Authorization: Bearer ${CLOUDFLARE_CACHE_TOKEN}" \ - -H "Content-Type: application/json" \ - --data "${json}" - - json="" - counter=0 - fi -done -if ! [[ "$counter" -eq 0 ]]; then - json="${json::-2}${NL}]}" - echo $json - - curl -X DELETE "https://api.cloudflare.com/client/v4/zones/${CLOUDFLARE_ZONE_ID}/purge_cache" \ - -H "X-Auth-Email: spring-sysadmin@pivotal.io" \ - -H "Authorization: Bearer ${CLOUDFLARE_CACHE_TOKEN}" \ - -H "Content-Type: application/json" \ - --data "${json}" -fi +#s3_url=s3://dist.springsource.com/${s3_path} +#files=`aws s3 cp ${s3_url} . --recursive --include "*" --dryrun` +#counter=0 +#json="" +#NL=$'\n' +#FILES_BATCH=7 +#for file in $files +#do +# if [[ "$counter" -eq 0 ]]; then +# json="{\"files\": [${NL}" +# fi +# if [[ "$file" =~ ^"s3://dist.springsource.com" ]]; then +# echo "Processing ${file}" +# counter=$((counter+1)) +# path=${file:26} +# json="${json}\"http://dist.springsource.com${path}\",${NL}\"https://dist.springsource.com${path}\",${NL}\"http://download.springsource.com${path}\",${NL}\"https://download.springsource.com${path}\",${NL}" +# fi +# if [[ "$counter" -eq "$FILES_BATCH" ]]; then +# json="${json::-2}${NL}]}" +# echo $json +# +# curl -X DELETE "https://api.cloudflare.com/client/v4/zones/${CLOUDFLARE_ZONE_ID}/purge_cache" \ +# -H "X-Auth-Email: spring-sysadmin@pivotal.io" \ +# -H "Authorization: Bearer ${CLOUDFLARE_CACHE_TOKEN}" \ +# -H "Content-Type: application/json" \ +# --data "${json}" +# +# json="" +# counter=0 +# fi +#done +#if ! [[ "$counter" -eq 0 ]]; then +# json="${json::-2}${NL}]}" +# echo $json +# +# curl -X DELETE "https://api.cloudflare.com/client/v4/zones/${CLOUDFLARE_ZONE_ID}/purge_cache" \ +# -H "X-Auth-Email: spring-sysadmin@pivotal.io" \ +# -H "Authorization: Bearer ${CLOUDFLARE_CACHE_TOKEN}" \ +# -H "Content-Type: application/json" \ +# --data "${json}" +#fi diff --git a/.github/workflows/clear_s3_cache.yml b/.github/workflows/clear_s3_cache.yml index 48d4f6b11..bf937454b 100644 --- a/.github/workflows/clear_s3_cache.yml +++ b/.github/workflows/clear_s3_cache.yml @@ -26,5 +26,4 @@ jobs: CLOUDFLARE_ZONE_ID: ${{ secrets.CLOUDFLARE_ZONE_ID }} CLOUDFLARE_CACHE_TOKEN: ${{ secrets.CLOUDFLARE_CACHE_TOKEN }} run: | - aws cloudfront create-invalidation --distribution-id ECAO9Q8651L8M --paths '/release/TOOLS/sts4-language-server-integrations/2023-07-31/*' ${{ github.workspace }}/.github/scripts/clear-s3-caches.sh ${{ inputs.path }} From 7cd8d66d3540f5eda961edec637d11fe428b3da4 Mon Sep 17 00:00:00 2001 From: aboyko Date: Mon, 31 Jul 2023 19:29:25 -0400 Subject: [PATCH 42/51] [GHA-Build] Correct release distros workflow --- .../release-eclipse-distro-build.yml | 45 +++++++++---------- 1 file changed, 21 insertions(+), 24 deletions(-) diff --git a/.github/workflows/release-eclipse-distro-build.yml b/.github/workflows/release-eclipse-distro-build.yml index fe0ca5da9..49328f446 100644 --- a/.github/workflows/release-eclipse-distro-build.yml +++ b/.github/workflows/release-eclipse-distro-build.yml @@ -15,31 +15,28 @@ on: jobs: e427-distro: - runs-on: ubuntu-latest - steps: - - uses: ./.github/workflows/gh-hosted-eclipse-distro-build.yml - with: - eclipse_profile: 'e427' - build_type: 'release' - p2-qualifier: 'RELEASE' - sts4-language-servers-version: ${{ inputs.sts4-language-servers-version }} + uses: ./.github/workflows/gh-hosted-eclipse-distro-build.yml + with: + eclipse_profile: 'e427' + build_type: 'release' + p2-qualifier: 'RELEASE' + sts4-language-servers-version: ${{ inputs.sts4-language-servers-version }} + secrets: inherit e428-distro: - runs-on: ubuntu-latest - steps: - - uses: ./.github/workflows/gh-hosted-eclipse-distro-build.yml - with: - eclipse_profile: 'e428' - build_type: 'release' - p2-qualifier: 'RELEASE' - sts4-language-servers-version: ${{ inputs.sts4-language-servers-version }} + uses: ./.github/workflows/gh-hosted-eclipse-distro-build.yml + with: + eclipse_profile: 'e428' + build_type: 'release' + p2-qualifier: 'RELEASE' + sts4-language-servers-version: ${{ inputs.sts4-language-servers-version }} + secrets: inherit e429-distro: - runs-on: ubuntu-latest - steps: - - uses: ./.github/workflows/gh-hosted-eclipse-distro-build.yml - with: - eclipse_profile: 'e429' - build_type: 'release' - p2-qualifier: 'RELEASE' - sts4-language-servers-version: ${{ inputs.sts4-language-servers-version }} + uses: ./.github/workflows/gh-hosted-eclipse-distro-build.yml + with: + eclipse_profile: 'e429' + build_type: 'release' + p2-qualifier: 'RELEASE' + sts4-language-servers-version: ${{ inputs.sts4-language-servers-version }} + secrets: inherit From 849051e62e923222acb53b151522791c28a2f944 Mon Sep 17 00:00:00 2001 From: aboyko Date: Mon, 31 Jul 2023 19:36:49 -0400 Subject: [PATCH 43/51] [GHA-Build] Putting everything together for release builds and cache clearance --- .github/scripts/clear-s3-caches.sh | 84 ++++++++++--------- .../workflows/eclipse-ls-extensions-build.yml | 2 +- .../gh-hosted-eclipse-distro-build.yml | 4 +- 3 files changed, 46 insertions(+), 44 deletions(-) diff --git a/.github/scripts/clear-s3-caches.sh b/.github/scripts/clear-s3-caches.sh index 8a665a675..d1844e0b7 100755 --- a/.github/scripts/clear-s3-caches.sh +++ b/.github/scripts/clear-s3-caches.sh @@ -2,6 +2,8 @@ set -e s3_path=$1 +echo "Clearing S3 caches for path: ${s3_path}" + #Flush AWS Cloudfront Cache invalidation_json=`aws cloudfront create-invalidation --distribution-id ECAO9Q8651L8M --output json --paths "/${s3_path}/*"` echo "Invalidation response: ${invalidation_json}" @@ -17,46 +19,46 @@ done echo "Final invalidation status: ${invalidation_status}" # Flush CloudFlare Cache -#s3_url=s3://dist.springsource.com/${s3_path} -#files=`aws s3 cp ${s3_url} . --recursive --include "*" --dryrun` -#counter=0 -#json="" -#NL=$'\n' -#FILES_BATCH=7 -#for file in $files -#do -# if [[ "$counter" -eq 0 ]]; then -# json="{\"files\": [${NL}" -# fi -# if [[ "$file" =~ ^"s3://dist.springsource.com" ]]; then -# echo "Processing ${file}" -# counter=$((counter+1)) -# path=${file:26} -# json="${json}\"http://dist.springsource.com${path}\",${NL}\"https://dist.springsource.com${path}\",${NL}\"http://download.springsource.com${path}\",${NL}\"https://download.springsource.com${path}\",${NL}" -# fi -# if [[ "$counter" -eq "$FILES_BATCH" ]]; then -# json="${json::-2}${NL}]}" -# echo $json -# -# curl -X DELETE "https://api.cloudflare.com/client/v4/zones/${CLOUDFLARE_ZONE_ID}/purge_cache" \ -# -H "X-Auth-Email: spring-sysadmin@pivotal.io" \ -# -H "Authorization: Bearer ${CLOUDFLARE_CACHE_TOKEN}" \ -# -H "Content-Type: application/json" \ -# --data "${json}" -# -# json="" -# counter=0 -# fi -#done -#if ! [[ "$counter" -eq 0 ]]; then -# json="${json::-2}${NL}]}" -# echo $json -# -# curl -X DELETE "https://api.cloudflare.com/client/v4/zones/${CLOUDFLARE_ZONE_ID}/purge_cache" \ -# -H "X-Auth-Email: spring-sysadmin@pivotal.io" \ -# -H "Authorization: Bearer ${CLOUDFLARE_CACHE_TOKEN}" \ -# -H "Content-Type: application/json" \ -# --data "${json}" -#fi +s3_url=s3://dist.springsource.com/${s3_path} +files=`aws s3 cp ${s3_url} . --recursive --include "*" --dryrun` +counter=0 +json="" +NL=$'\n' +FILES_BATCH=7 +for file in $files +do + if [[ "$counter" -eq 0 ]]; then + json="{\"files\": [${NL}" + fi + if [[ "$file" =~ ^"s3://dist.springsource.com" ]]; then + echo "Processing ${file}" + counter=$((counter+1)) + path=${file:26} + json="${json}\"http://dist.springsource.com${path}\",${NL}\"https://dist.springsource.com${path}\",${NL}\"http://download.springsource.com${path}\",${NL}\"https://download.springsource.com${path}\",${NL}" + fi + if [[ "$counter" -eq "$FILES_BATCH" ]]; then + json="${json::-2}${NL}]}" + echo $json + + curl -X DELETE "https://api.cloudflare.com/client/v4/zones/${CLOUDFLARE_ZONE_ID}/purge_cache" \ + -H "X-Auth-Email: spring-sysadmin@pivotal.io" \ + -H "Authorization: Bearer ${CLOUDFLARE_CACHE_TOKEN}" \ + -H "Content-Type: application/json" \ + --data "${json}" + + json="" + counter=0 + fi +done +if ! [[ "$counter" -eq 0 ]]; then + json="${json::-2}${NL}]}" + echo $json + + curl -X DELETE "https://api.cloudflare.com/client/v4/zones/${CLOUDFLARE_ZONE_ID}/purge_cache" \ + -H "X-Auth-Email: spring-sysadmin@pivotal.io" \ + -H "Authorization: Bearer ${CLOUDFLARE_CACHE_TOKEN}" \ + -H "Content-Type: application/json" \ + --data "${json}" +fi diff --git a/.github/workflows/eclipse-ls-extensions-build.yml b/.github/workflows/eclipse-ls-extensions-build.yml index 3d8400530..265e5e35a 100644 --- a/.github/workflows/eclipse-ls-extensions-build.yml +++ b/.github/workflows/eclipse-ls-extensions-build.yml @@ -89,7 +89,7 @@ jobs: CLOUDFLARE_CACHE_TOKEN: ${{ secrets.CLOUDFLARE_CACHE_TOKEN }} run: | dist_path=`cat ${{ github.workspace }}/eclipse-language-servers/org.springframework.tooling.ls.integration.repository/target/s3-p2-repo-dist-path.txt` - echo $dist_path + ${{ github.workspace }}/.github/scripts/clear-s3-caches.sh $dist_path - name: Verify Eclipse LS Extensions on e429 if: ${{ inputs.build_type == 'snapshot' }} run: | diff --git a/.github/workflows/gh-hosted-eclipse-distro-build.yml b/.github/workflows/gh-hosted-eclipse-distro-build.yml index 18a83e06d..8f633baea 100644 --- a/.github/workflows/gh-hosted-eclipse-distro-build.yml +++ b/.github/workflows/gh-hosted-eclipse-distro-build.yml @@ -100,7 +100,7 @@ jobs: CLOUDFLARE_CACHE_TOKEN: ${{ secrets.CLOUDFLARE_CACHE_TOKEN }} run: | dist_path=`cat ${{ github.workspace }}/eclipse-distribution/org.springframework.boot.ide.product.${{ inputs.eclipse_profile }}/target/s3-p2-repo-dist-path.txt` - echo $dist_path + ${{ github.workspace }}/.github/scripts/clear-s3-caches.sh $dist_path - name: Update Nightly Distro Downloads page if: ${{ inputs.build_type == 'snapshot' && always() }} env: @@ -308,7 +308,7 @@ jobs: CLOUDFLARE_CACHE_TOKEN: ${{ secrets.CLOUDFLARE_CACHE_TOKEN }} run: | dist_path=`cat s3-dist-path.txt` - echo $dist_path + ${{ github.workspace }}/.github/scripts/clear-s3-caches.sh $dist_path - name: Remove Temp Build Artifacts from S3 id: cleanup-s3-temp-storage env: From 49055bc014b1c4426e7b64e25f87b5418e3636a1 Mon Sep 17 00:00:00 2001 From: aboyko Date: Mon, 31 Jul 2023 20:07:01 -0400 Subject: [PATCH 44/51] [GHA-Build] Correct osx vs linux incompatibility --- .github/scripts/clear-s3-caches.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/scripts/clear-s3-caches.sh b/.github/scripts/clear-s3-caches.sh index d1844e0b7..56d28289e 100755 --- a/.github/scripts/clear-s3-caches.sh +++ b/.github/scripts/clear-s3-caches.sh @@ -37,7 +37,7 @@ do json="${json}\"http://dist.springsource.com${path}\",${NL}\"https://dist.springsource.com${path}\",${NL}\"http://download.springsource.com${path}\",${NL}\"https://download.springsource.com${path}\",${NL}" fi if [[ "$counter" -eq "$FILES_BATCH" ]]; then - json="${json::-2}${NL}]}" + json="${json:0:-2}${NL}]}" echo $json curl -X DELETE "https://api.cloudflare.com/client/v4/zones/${CLOUDFLARE_ZONE_ID}/purge_cache" \ @@ -51,7 +51,7 @@ do fi done if ! [[ "$counter" -eq 0 ]]; then - json="${json::-2}${NL}]}" + json="${json:0:-2}${NL}]}" echo $json curl -X DELETE "https://api.cloudflare.com/client/v4/zones/${CLOUDFLARE_ZONE_ID}/purge_cache" \ From 004da9a70df1532dd329bfad354a236360683d94 Mon Sep 17 00:00:00 2001 From: aboyko Date: Mon, 31 Jul 2023 20:07:33 -0400 Subject: [PATCH 45/51] [GHA-Build] Correct osx vs linux incompatibility --- .github/workflows/clear_s3_cache.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/clear_s3_cache.yml b/.github/workflows/clear_s3_cache.yml index bf937454b..a3a525727 100644 --- a/.github/workflows/clear_s3_cache.yml +++ b/.github/workflows/clear_s3_cache.yml @@ -11,7 +11,7 @@ on: jobs: clear-s3-cache: - runs-on: ubuntu-latest + runs-on: macos-latest steps: - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 with: From 17ce12576080ccc7ce7cfeaa7f99f277fd5db6b7 Mon Sep 17 00:00:00 2001 From: aboyko Date: Mon, 31 Jul 2023 20:11:18 -0400 Subject: [PATCH 46/51] [GHA-Build] Correct osx vs linux incompatibility --- .github/scripts/clear-s3-caches.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/scripts/clear-s3-caches.sh b/.github/scripts/clear-s3-caches.sh index 56d28289e..4879c445a 100755 --- a/.github/scripts/clear-s3-caches.sh +++ b/.github/scripts/clear-s3-caches.sh @@ -37,7 +37,7 @@ do json="${json}\"http://dist.springsource.com${path}\",${NL}\"https://dist.springsource.com${path}\",${NL}\"http://download.springsource.com${path}\",${NL}\"https://download.springsource.com${path}\",${NL}" fi if [[ "$counter" -eq "$FILES_BATCH" ]]; then - json="${json:0:-2}${NL}]}" + json="${json%,*}${NL}]}" echo $json curl -X DELETE "https://api.cloudflare.com/client/v4/zones/${CLOUDFLARE_ZONE_ID}/purge_cache" \ @@ -51,7 +51,7 @@ do fi done if ! [[ "$counter" -eq 0 ]]; then - json="${json:0:-2}${NL}]}" + json="${json%,*}${NL}]}" echo $json curl -X DELETE "https://api.cloudflare.com/client/v4/zones/${CLOUDFLARE_ZONE_ID}/purge_cache" \ From 4cb9b63c0f4eadbc7c336c30b5d94edd9027af5d Mon Sep 17 00:00:00 2001 From: aboyko Date: Mon, 31 Jul 2023 21:35:38 -0400 Subject: [PATCH 47/51] [GHA-Build] Fix usage of workflows within other workflow --- .../gh-hosted-eclipse-distro-build.yml | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/.github/workflows/gh-hosted-eclipse-distro-build.yml b/.github/workflows/gh-hosted-eclipse-distro-build.yml index 8f633baea..aead9665a 100644 --- a/.github/workflows/gh-hosted-eclipse-distro-build.yml +++ b/.github/workflows/gh-hosted-eclipse-distro-build.yml @@ -118,16 +118,19 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLS_S3_SECRET_KEY }} run: | echo "Uploading Win Zips and OSX tar.gz to S3 for signing..." - aws s3 cp ${{ github.workspace }}/eclipse-distribution/org.springframework.boot.ide.product.${{ inputs.eclipse_profile }}/target/products s3://dist.springsource.com/sts4-distro-ci-temp/${{ github.run_id }} --recursive --exclude "*" --include "spring-tool-suite-4*win*.zip" --include "spring-tool-suite-4*macosx*.tar.gz" --exclude "*/*" --no-progress + id=${{ inputs.eclipse_profile }}-${{ inputs.build_type }}-${{ github.run_id }} + aws s3 cp ${{ github.workspace }}/eclipse-distribution/org.springframework.boot.ide.product.${{ inputs.eclipse_profile }}/target/products s3://dist.springsource.com/sts4-distro-ci-temp/$id --recursive --exclude "*" --include "spring-tool-suite-4*win*.zip" --include "spring-tool-suite-4*macosx*.tar.gz" --exclude "*/*" --no-progress - uses: actions/upload-artifact@0b7f8abb1508181956e8e162db84b466c27e18ce with: - name: s3-dist-path-${{ github.run_id }}.${{ github.run_attempt }} + name: s3-dist-path-${{ inputs.eclipse_profile }}-${{ inputs.build_type }}-${{ github.run_id }} path: ${{ github.workspace }}/eclipse-distribution/org.springframework.boot.ide.product.${{ inputs.eclipse_profile }}/target/s3-dist-path.txt retention-days: 1 + outputs: + id: ${{ inputs.eclipse_profile }}-${{ inputs.build_type }}-${{ github.run_id }} sign-win-distros: - needs: eclipse-distro-build + needs: [ eclipse-distro-build ] runs-on: self-hosted steps: - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 @@ -142,7 +145,7 @@ jobs: rm -f spring-tool-suite-4*win*.zip* rm -f spring-tool-suite-4*win*.self-extracting.jar* ls - aws s3 mv s3://dist.springsource.com/sts4-distro-ci-temp/${{ github.run_id }} . --recursive --exclude "*" --include "spring-tool-suite-4*win*.zip*" --exclude "*/*" --no-progress + aws s3 mv s3://dist.springsource.com/sts4-distro-ci-temp/${{ needs.eclipse-distro-build.outputs.id }} . --recursive --exclude "*" --include "spring-tool-suite-4*win*.zip*" --exclude "*/*" --no-progress - name: Sign EXE within zip files id: sign env: @@ -152,11 +155,11 @@ jobs: files=`ls spring-tool-suite-4*win*.zip` for file in $files do - ${{ github.workspace }}/.github/scripts/sign-exe-in-zip-file.sh $file ${{ github.workspace }}/.github/scripts/sign-exe.sh ${{ github.workspace }}/.github/scripts/self-extracting-jar-creator.jar ${{ github.run_id }} + ${{ github.workspace }}/.github/scripts/sign-exe-in-zip-file.sh $file ${{ github.workspace }}/.github/scripts/sign-exe.sh ${{ github.workspace }}/.github/scripts/self-extracting-jar-creator.jar ${{ needs.eclipse-distro-build.outputs.id }} done - uses: actions/download-artifact@cbed621e49e4c01b044d60f6c80ea4ed6328b281 with: - name: s3-dist-path-${{ github.run_id }}.${{ github.run_attempt }} + name: s3-dist-path-${{ needs.eclipse-distro-build.outputs.id }} - name: Update Win zip/jar on S3 id: update-s3 env: @@ -188,11 +191,11 @@ jobs: SSH_KEY: ~/.ssh/id_rsa SSH_USER: signer run: | - ssh -i $SSH_KEY $SSH_USER@vm-tools.spring.vmware.com -- rm -rf /opt/bamboo/${{ github.run_id }} + ssh -i $SSH_KEY $SSH_USER@vm-tools.spring.vmware.com -- rm -rf /opt/bamboo/${{ needs.eclipse-distro-build.outputs.id }} rm -rf *spring-tool-suite-4*win* sign-osx-distros: - needs: eclipse-distro-build + needs: [ eclipse-distro-build ] runs-on: macos-latest-xl steps: - uses: actions/checkout@c85c95e3d7251135ab7dc9ce3241c5835cc595a9 @@ -229,7 +232,7 @@ jobs: run: | rm -rf *macosx* ls - aws s3 mv s3://dist.springsource.com/sts4-distro-ci-temp/${{ github.run_id }} . --recursive --exclude "*" --include "spring-tool-suite-4*macosx*.tar.gz" --exclude "*/*" --no-progress + aws s3 mv s3://dist.springsource.com/sts4-distro-ci-temp/${{ needs.eclipse-distro-build.outputs.id }} . --recursive --exclude "*" --include "spring-tool-suite-4*macosx*.tar.gz" --exclude "*/*" --no-progress - name: Sign .app, Create and Sign DMG env: MACOS_CERTIFICATE_ID: ${{ secrets.MACOS_CERTIFICATE_ID }} @@ -256,7 +259,7 @@ jobs: echo "Done signing and notarization of DMG files" - uses: actions/download-artifact@cbed621e49e4c01b044d60f6c80ea4ed6328b281 with: - name: s3-dist-path-${{ github.run_id }}.${{ github.run_attempt }} + name: s3-dist-path-${{ needs.eclipse-distro-build.outputs.id }} - name: Update Win zip/jar on S3 id: update-s3 env: @@ -287,7 +290,7 @@ jobs: rm -rf *spring-tool-suite-4*macosx* cleanup: - needs: [ sign-win-distros, sign-osx-distros ] + needs: [ eclipse-distro-build, sign-win-distros, sign-osx-distros ] if: ${{ always() }} runs-on: ubuntu-latest steps: @@ -297,7 +300,7 @@ jobs: .github - uses: actions/download-artifact@cbed621e49e4c01b044d60f6c80ea4ed6328b281 with: - name: s3-dist-path-${{ github.run_id }}.${{ github.run_attempt }} + name: s3-dist-path-${{ needs.eclipse-distro-build.outputs.id }} - name: Clear S3 Caches for Distros if: ${{ inputs.build_type != 'snapshot' }} env: @@ -316,4 +319,4 @@ jobs: AWS_SECRET_ACCESS_KEY: ${{ secrets.TOOLS_S3_SECRET_KEY }} AWS_DEFAULT_REGION: us-east-1 run: | - aws s3 rm s3://dist.springsource.com/sts4-distro-ci-temp --recursive --exclude "*" --include "${{ github.run_id }}/*" + aws s3 rm s3://dist.springsource.com/sts4-distro-ci-temp --recursive --exclude "*" --include "${{ needs.eclipse-distro-build.outputs.id }}/*" From 419597df7980a1a65b22bac7152bc6b21adaca48 Mon Sep 17 00:00:00 2001 From: aboyko Date: Mon, 31 Jul 2023 23:11:19 -0400 Subject: [PATCH 48/51] Lock rewrite versions for release preparation --- .../workflows/release-eclipse-ls-extensions.yml | 17 ++++++----------- headless-services/commons/pom.xml | 6 +++--- 2 files changed, 9 insertions(+), 14 deletions(-) diff --git a/.github/workflows/release-eclipse-ls-extensions.yml b/.github/workflows/release-eclipse-ls-extensions.yml index e68595223..508fac7c4 100644 --- a/.github/workflows/release-eclipse-ls-extensions.yml +++ b/.github/workflows/release-eclipse-ls-extensions.yml @@ -6,24 +6,19 @@ concurrency: on: workflow_dispatch: + inputs: + sts4-language-servers-version: + description: Last segments of an s3 path for p2 Repo for eclipse ls extensions + required: true + type: string jobs: - generate-sts4-ls-extensions-version: - runs-on: ubuntu-latest - steps: - - name: Timestamp - id: timestamp - run: echo "date=$(date +'%Y-%m-%d')" >> $GITHUB_OUTPUT - outputs: - sts4-language-servers-version: ${{ steps.timestamp.outputs.date }} - eclipse-ls-extension: - needs: [ generate-sts4-ls-extensions-version ] uses: ./.github/workflows/eclipse-ls-extensions-build.yml with: eclipse_profile: 'e427' build_type: 'release' skip_tests: true - sts4-language-servers-version: ${{ needs.generate-sts4-ls-extensions-version.outputs.sts4-language-servers-version }} + sts4-language-servers-version: ${{ inputs.sts4-language-servers-version }} secrets: inherit diff --git a/headless-services/commons/pom.xml b/headless-services/commons/pom.xml index 3ea966fcb..5cadfbf01 100644 --- a/headless-services/commons/pom.xml +++ b/headless-services/commons/pom.xml @@ -113,9 +113,9 @@ 1.13 - 8.2.0-SNAPSHOT - 5.1.0-SNAPSHOT - 1.2.0-SNAPSHOT + 8.1.13 + 5.0.6 + 1.1.2 7.6 6.1.1 From 77078a426c56f9a7820c112cbb57e64bb2b5a448 Mon Sep 17 00:00:00 2001 From: aboyko Date: Mon, 31 Jul 2023 23:45:37 -0400 Subject: [PATCH 49/51] GHA: Back to snapshot build on every push to main branch --- .../workflows/snapshot-eclipse-ls-extensions-build.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/snapshot-eclipse-ls-extensions-build.yml b/.github/workflows/snapshot-eclipse-ls-extensions-build.yml index c0307c642..f398da152 100644 --- a/.github/workflows/snapshot-eclipse-ls-extensions-build.yml +++ b/.github/workflows/snapshot-eclipse-ls-extensions-build.yml @@ -4,12 +4,12 @@ concurrency: group: eclipse-ls-extension-snapshot cancel-in-progress: true -on: - workflow_dispatch: #on: -# push: -# branches: -# - 'main' +# workflow_dispatch: +on: + push: + branches: + - 'main' jobs: eclipse-ls-extensions: From 0d26714a03ce39cd1320624746afed797bf16329 Mon Sep 17 00:00:00 2001 From: aboyko Date: Mon, 31 Jul 2023 23:47:51 -0400 Subject: [PATCH 50/51] GHA: Adjust label on one of the build steps --- .github/workflows/gh-hosted-eclipse-distro-build.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/gh-hosted-eclipse-distro-build.yml b/.github/workflows/gh-hosted-eclipse-distro-build.yml index aead9665a..e2639bc74 100644 --- a/.github/workflows/gh-hosted-eclipse-distro-build.yml +++ b/.github/workflows/gh-hosted-eclipse-distro-build.yml @@ -260,7 +260,7 @@ jobs: - uses: actions/download-artifact@cbed621e49e4c01b044d60f6c80ea4ed6328b281 with: name: s3-dist-path-${{ needs.eclipse-distro-build.outputs.id }} - - name: Update Win zip/jar on S3 + - name: Update GMG files on S3 id: update-s3 env: AWS_ACCESS_KEY_ID: ${{ secrets.TOOLS_S3_ACCESS_KEY }} From 3b82f4bb2f31bb822c21d8fbb6042e5ef362ed2b Mon Sep 17 00:00:00 2001 From: Martin Lippert Date: Tue, 1 Aug 2023 10:10:53 +0200 Subject: [PATCH 51/51] GH-1056: no more duplicated symbols being produced due to disconnected removal of symbols on project initialization and change --- .../ide/vscode/boot/app/SpringSymbolIndex.java | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/headless-services/spring-boot-language-server/src/main/java/org/springframework/ide/vscode/boot/app/SpringSymbolIndex.java b/headless-services/spring-boot-language-server/src/main/java/org/springframework/ide/vscode/boot/app/SpringSymbolIndex.java index 41e6758a5..3354f9df2 100644 --- a/headless-services/spring-boot-language-server/src/main/java/org/springframework/ide/vscode/boot/app/SpringSymbolIndex.java +++ b/headless-services/spring-boot-language-server/src/main/java/org/springframework/ide/vscode/boot/app/SpringSymbolIndex.java @@ -365,14 +365,19 @@ public class SpringSymbolIndex implements InitializingBean, SpringIndex { return CompletableFuture.completedFuture(null); } else { - removeSymbolsByProject(project); - springIndex.removeBeans(project.getElementName()); - @SuppressWarnings("unchecked") - CompletableFuture[] futures = new CompletableFuture[this.indexers.length]; + CompletableFuture[] futures = new CompletableFuture[this.indexers.length + 1]; + + // clean future + futures[0] = CompletableFuture.runAsync(() -> { + removeSymbolsByProject(project); + springIndex.removeBeans(project.getElementName()); + }, this.updateQueue); + + // index futures for (int i = 0; i < this.indexers.length; i++) { InitializeProject initializeItem = new InitializeProject(project, this.indexers[i]); - futures[i] = CompletableFuture.runAsync(initializeItem, this.updateQueue); + futures[i + 1] = CompletableFuture.runAsync(initializeItem, this.updateQueue); } CompletableFuture future = CompletableFuture.allOf(futures);