Spring/spring-boot
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Check if promotion has already occurred

Browse Source 
Update the release tooling to check for bintray published artifacts
using SHA256 digests and to also check before attempting a promote.

See gh-21474
This commit is contained in:
Phillip Webb
2020-06-11 21:03:01 -07:00
parent d650c5fdf2
commit ce011ca384
11 changed files with 189 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
ci/images/releasescripts/src/main/java/io/spring/concourse/releasescripts/artifactory/ArtifactoryService.java
View File

@@ -17,6 +17,8 @@
package io.spring.concourse.releasescripts.artifactory;
import java.net.URI;
import java.time.Duration;
import java.util.Set;
import io.spring.concourse.releasescripts.ReleaseInfo;
import io.spring.concourse.releasescripts.artifactory.payload.BuildInfoResponse;
@@ -113,8 +115,12 @@ public class ArtifactoryService {
* Deploy builds from Artifactory to Bintray.
* @param sourceRepo the source repo in Artifactory.
* @param releaseInfo the resease info
* @param artifactDigests the artifact digests
*/
public void distribute(String sourceRepo, ReleaseInfo releaseInfo) {
public void distribute(String sourceRepo, ReleaseInfo releaseInfo, Set<String> artifactDigests) {
if (this.bintrayService.isDistributionComplete(releaseInfo, artifactDigests, Duration.ofMinutes(2))) {
console.log("Distribution already complete");
}
DistributionRequest request = new DistributionRequest(new String[] { sourceRepo });
RequestEntity<DistributionRequest> requestEntity = RequestEntity
.post(URI.create(DISTRIBUTION_URL + releaseInfo.getBuildName() + "/" + releaseInfo.getBuildNumber()))
@@ -126,7 +132,7 @@ public class ArtifactoryService {
console.log("Failed to distribute.");
throw ex;
}
if (!this.bintrayService.isDistributionComplete(releaseInfo)) {
if (!this.bintrayService.isDistributionComplete(releaseInfo, artifactDigests, Duration.ofMinutes(60))) {
throw new DistributionTimeoutException("Distribution timed out.");
}

49
ci/images/releasescripts/src/main/java/io/spring/concourse/releasescripts/artifactory/payload/BuildInfoResponse.java
View File

@@ -16,6 +16,11 @@
package io.spring.concourse.releasescripts.artifactory.payload;
import java.util.Arrays;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
/**
* Represents the response from Artifactory's buildInfo endpoint.
*
@@ -54,7 +59,7 @@ public class BuildInfoResponse {
}
public String getName() {
return name;
return this.name;
}
public void setName(String name) {
@@ -83,6 +88,14 @@ public class BuildInfoResponse {
public void setVersion(String version) {
this.version = version;
}
public Set<String> getArtifactDigests() {
return Arrays.stream(this.modules).flatMap((module) -> {
Artifact[] artifacts = module.getArtifacts();
return (artifacts != null) ? Arrays.stream(artifacts) : Stream.empty();
}).map(Artifact::getSha256).collect(Collectors.toSet());
}
}
@@ -105,6 +118,8 @@ public class BuildInfoResponse {
private String id;
private Artifact[] artifacts;
public String getId() {
return this.id;
}
@@ -113,6 +128,38 @@ public class BuildInfoResponse {
this.id = id;
}
public Artifact[] getArtifacts() {
return this.artifacts;
}
public void setArtifacts(Artifact[] artifacts) {
this.artifacts = artifacts;
}
}
public static class Artifact {
private String name;
private String sha256;
public String getName() {
return this.name;
}
public void setName(String name) {
this.name = name;
}
public String getSha256() {
return this.sha256;
}
public void setSha256(String sha256) {
this.sha256 = sha256;
}
}
}

44
ci/images/releasescripts/src/main/java/io/spring/concourse/releasescripts/bintray/BintrayService.java
View File

@@ -17,8 +17,9 @@
package io.spring.concourse.releasescripts.bintray;
import java.net.URI;
import java.util.Objects;
import java.util.concurrent.TimeUnit;
import java.time.Duration;
import java.util.HashSet;
import java.util.Set;
import io.spring.concourse.releasescripts.ReleaseInfo;
import io.spring.concourse.releasescripts.sonatype.SonatypeProperties;
@@ -27,7 +28,6 @@ import io.spring.concourse.releasescripts.system.ConsoleLogger;
import org.awaitility.core.ConditionTimeoutException;
import org.springframework.boot.web.client.RestTemplateBuilder;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.RequestEntity;
import org.springframework.stereotype.Component;
@@ -72,24 +72,17 @@ public class BintrayService {
this.restTemplate = builder.build();
}
public boolean isDistributionComplete(ReleaseInfo releaseInfo) {
RequestEntity<Void> allFilesRequest = getRequest(releaseInfo, 1);
Object[] allFiles = waitAtMost(5, TimeUnit.MINUTES).with().pollDelay(20, TimeUnit.SECONDS).until(() -> {
try {
return this.restTemplate.exchange(allFilesRequest, Object[].class).getBody();
}
catch (HttpClientErrorException ex) {
if (ex.getStatusCode() != HttpStatus.NOT_FOUND) {
throw ex;
}
return null;
}
}, Objects::nonNull);
RequestEntity<Void> publishedFilesRequest = getRequest(releaseInfo, 0);
public boolean isDistributionComplete(ReleaseInfo releaseInfo, Set<String> requiredDigets, Duration timeout) {
return isDistributionComplete(releaseInfo, requiredDigets, timeout, Duration.ofSeconds(20));
}
public boolean isDistributionComplete(ReleaseInfo releaseInfo, Set<String> requiredDigets, Duration timeout,
Duration pollDelay) {
RequestEntity<Void> request = getRequest(releaseInfo, 0);
try {
waitAtMost(120, TimeUnit.MINUTES).with().pollDelay(20, TimeUnit.SECONDS).until(() -> {
Object[] publishedFiles = this.restTemplate.exchange(publishedFilesRequest, Object[].class).getBody();
return allFiles.length == publishedFiles.length;
waitAtMost(timeout).with().pollDelay(pollDelay).until(() -> {
PackageFile[] published = this.restTemplate.exchange(request, PackageFile[].class).getBody();
return hasPublishedAll(published, requiredDigets);
});
}
catch (ConditionTimeoutException ex) {
@@ -98,6 +91,17 @@ public class BintrayService {
return true;
}
private boolean hasPublishedAll(PackageFile[] published, Set<String> requiredDigets) {
if (published == null || published.length == 0) {
return false;
}
Set<String> remaining = new HashSet<>(requiredDigets);
for (PackageFile publishedFile : published) {
remaining.remove(publishedFile.getSha256());
}
return remaining.isEmpty();
}
private RequestEntity<Void> getRequest(ReleaseInfo releaseInfo, int includeUnpublished) {
return RequestEntity.get(URI.create(BINTRAY_URL + "packages/" + this.bintrayProperties.getSubject() + "/"
+ this.bintrayProperties.getRepo() + "/" + releaseInfo.getGroupId() + "/versions/"

38
ci/images/releasescripts/src/main/java/io/spring/concourse/releasescripts/bintray/PackageFile.java Normal file
View File

@@ -0,0 +1,38 @@
/*
* Copyright 2020 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.spring.concourse.releasescripts.bintray;
/**
* Details for a single packaged file.
*
* @author Phillip Webb
*/
public class PackageFile {
private String name;
private String sha256;
public String getName() {
return this.name;
}
public String getSha256() {
return this.sha256;
}
}

14
ci/images/releasescripts/src/main/java/io/spring/concourse/releasescripts/command/DistributeCommand.java
View File

@@ -19,12 +19,14 @@ package io.spring.concourse.releasescripts.command;
import java.io.File;
import java.nio.file.Files;
import java.util.List;
import java.util.Set;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.spring.concourse.releasescripts.ReleaseInfo;
import io.spring.concourse.releasescripts.ReleaseType;
import io.spring.concourse.releasescripts.artifactory.ArtifactoryService;
import io.spring.concourse.releasescripts.artifactory.payload.BuildInfoResponse;
import io.spring.concourse.releasescripts.artifactory.payload.BuildInfoResponse.BuildInfo;
import org.springframework.boot.ApplicationArguments;
import org.springframework.stereotype.Component;
@@ -38,12 +40,12 @@ import org.springframework.util.Assert;
@Component
public class DistributeCommand implements Command {
private final ArtifactoryService service;
private final ArtifactoryService artifactoryService;
private final ObjectMapper objectMapper;
public DistributeCommand(ArtifactoryService service, ObjectMapper objectMapper) {
this.service = service;
public DistributeCommand(ArtifactoryService artifactoryService, ObjectMapper objectMapper) {
this.artifactoryService = artifactoryService;
this.objectMapper = objectMapper;
}
@@ -60,8 +62,10 @@ public class DistributeCommand implements Command {
String buildInfoLocation = nonOptionArgs.get(2);
byte[] content = Files.readAllBytes(new File(buildInfoLocation).toPath());
BuildInfoResponse buildInfoResponse = this.objectMapper.readValue(content, BuildInfoResponse.class);
ReleaseInfo releaseInfo = ReleaseInfo.from(buildInfoResponse.getBuildInfo());
this.service.distribute(type.getRepo(), releaseInfo);
BuildInfo buildInfo = buildInfoResponse.getBuildInfo();
Set<String> artifactDigests = buildInfo.getArtifactDigests();
ReleaseInfo releaseInfo = ReleaseInfo.from(buildInfo);
this.artifactoryService.distribute(type.getRepo(), releaseInfo, artifactDigests);
}
}

43
ci/images/releasescripts/src/test/java/io/spring/concourse/releasescripts/artifactory/ArtifactoryServiceTests.java
View File

@@ -16,10 +16,15 @@
package io.spring.concourse.releasescripts.artifactory;
import java.time.Duration;
import java.util.Collections;
import java.util.Set;
import io.spring.concourse.releasescripts.ReleaseInfo;
import io.spring.concourse.releasescripts.bintray.BintrayService;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import org.mockito.InOrder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
@@ -35,10 +40,13 @@ import org.springframework.util.Base64Utils;
import org.springframework.web.client.HttpClientErrorException;
import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.BDDMockito.given;
import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.verifyNoMoreInteractions;
import static org.springframework.test.web.client.match.MockRestRequestMatchers.content;
import static org.springframework.test.web.client.match.MockRestRequestMatchers.header;
import static org.springframework.test.web.client.match.MockRestRequestMatchers.method;
@@ -55,6 +63,10 @@ import static org.springframework.test.web.client.response.MockRestResponseCreat
@EnableConfigurationProperties(ArtifactoryProperties.class)
class ArtifactoryServiceTests {
private static final Duration SHORT_TIMEOUT = Duration.ofMinutes(2);
private static final Duration LONG_TIMEOUT = Duration.ofMinutes(60);
@Autowired
private ArtifactoryService service;
@@ -118,9 +130,10 @@ class ArtifactoryServiceTests {
}
@Test
@SuppressWarnings("unchecked")
void distributeWhenSuccessful() throws Exception {
ReleaseInfo releaseInfo = getReleaseInfo();
given(this.bintrayService.isDistributionComplete(releaseInfo)).willReturn(true);
given(this.bintrayService.isDistributionComplete(eq(releaseInfo), (Set<String>) any(), any())).willReturn(true);
this.server.expect(requestTo("https://repo.spring.io/api/build/distribute/example-build/example-build-1"))
.andExpect(method(HttpMethod.POST))
.andExpect(content().json(
@@ -128,9 +141,12 @@ class ArtifactoryServiceTests {
.andExpect(header("Authorization", "Basic " + Base64Utils.encodeToString(String
.format("%s:%s", this.properties.getUsername(), this.properties.getPassword()).getBytes())))
.andExpect(header("Content-Type", MediaType.APPLICATION_JSON.toString())).andRespond(withSuccess());
this.service.distribute("libs-release-local", releaseInfo);
Set<String> artifactDigests = Collections.singleton("602e20176706d3cc7535f01ffdbe91b270ae5014");
this.service.distribute("libs-release-local", releaseInfo, artifactDigests);
this.server.verify();
verify(this.bintrayService, times(1)).isDistributionComplete(releaseInfo);
InOrder ordered = inOrder(this.bintrayService);
ordered.verify(this.bintrayService).isDistributionComplete(releaseInfo, artifactDigests, SHORT_TIMEOUT);
ordered.verify(this.bintrayService).isDistributionComplete(releaseInfo, artifactDigests, LONG_TIMEOUT);
}
@Test
@@ -144,16 +160,22 @@ class ArtifactoryServiceTests {
.format("%s:%s", this.properties.getUsername(), this.properties.getPassword()).getBytes())))
.andExpect(header("Content-Type", MediaType.APPLICATION_JSON.toString()))
.andRespond(withStatus(HttpStatus.FORBIDDEN));
Set<String> artifactDigests = Collections.singleton("602e20176706d3cc7535f01ffdbe91b270ae5014");
assertThatExceptionOfType(HttpClientErrorException.class)
.isThrownBy(() -> this.service.distribute("libs-release-local", releaseInfo));
.isThrownBy(() -> this.service.distribute("libs-release-local", releaseInfo, artifactDigests));
this.server.verify();
verifyNoInteractions(this.bintrayService);
verify(this.bintrayService, times(1)).isDistributionComplete(releaseInfo, artifactDigests, SHORT_TIMEOUT);
verifyNoMoreInteractions(this.bintrayService);
}
@Test
@SuppressWarnings("unchecked")
void distributeWhenGettingPackagesTimesOut() throws Exception {
ReleaseInfo releaseInfo = getReleaseInfo();
given(this.bintrayService.isDistributionComplete(releaseInfo)).willReturn(false);
given(this.bintrayService.isDistributionComplete(eq(releaseInfo), (Set<String>) any(), any()))
.willReturn(false);
given(this.bintrayService.isDistributionComplete(eq(releaseInfo), (Set<String>) any(), any()))
.willReturn(false);
this.server.expect(requestTo("https://repo.spring.io/api/build/distribute/example-build/example-build-1"))
.andExpect(method(HttpMethod.POST))
.andExpect(content().json(
@@ -161,10 +183,13 @@ class ArtifactoryServiceTests {
.andExpect(header("Authorization", "Basic " + Base64Utils.encodeToString(String
.format("%s:%s", this.properties.getUsername(), this.properties.getPassword()).getBytes())))
.andExpect(header("Content-Type", MediaType.APPLICATION_JSON.toString())).andRespond(withSuccess());
Set<String> artifactDigests = Collections.singleton("602e20176706d3cc7535f01ffdbe91b270ae5014");
assertThatExceptionOfType(DistributionTimeoutException.class)
.isThrownBy(() -> this.service.distribute("libs-release-local", releaseInfo));
.isThrownBy(() -> this.service.distribute("libs-release-local", releaseInfo, artifactDigests));
this.server.verify();
verify(this.bintrayService, times(1)).isDistributionComplete(releaseInfo);
InOrder ordered = inOrder(this.bintrayService);
ordered.verify(this.bintrayService).isDistributionComplete(releaseInfo, artifactDigests, SHORT_TIMEOUT);
ordered.verify(this.bintrayService).isDistributionComplete(releaseInfo, artifactDigests, LONG_TIMEOUT);
}
private ReleaseInfo getReleaseInfo() {

22
ci/images/releasescripts/src/test/java/io/spring/concourse/releasescripts/bintray/BintrayServiceTests.java
View File

@@ -16,6 +16,10 @@
package io.spring.concourse.releasescripts.bintray;
import java.time.Duration;
import java.util.LinkedHashSet;
import java.util.Set;
import io.spring.concourse.releasescripts.ReleaseInfo;
import io.spring.concourse.releasescripts.sonatype.SonatypeProperties;
import io.spring.concourse.releasescripts.sonatype.SonatypeService;
@@ -28,7 +32,6 @@ import org.springframework.boot.test.autoconfigure.web.client.RestClientTest;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.core.io.ClassPathResource;
import org.springframework.http.HttpMethod;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.test.web.client.ExpectedCount;
import org.springframework.test.web.client.MockRestServiceServer;
@@ -41,7 +44,6 @@ import static org.springframework.test.web.client.match.MockRestRequestMatchers.
import static org.springframework.test.web.client.match.MockRestRequestMatchers.header;
import static org.springframework.test.web.client.match.MockRestRequestMatchers.method;
import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo;
import static org.springframework.test.web.client.response.MockRestResponseCreators.withStatus;
import static org.springframework.test.web.client.response.MockRestResponseCreators.withSuccess;
/**
@@ -75,15 +77,15 @@ class BintrayServiceTests {
@Test
void isDistributionComplete() throws Exception {
this.server
.expect(requestTo(String.format(
"https://api.bintray.com/packages/%s/%s/%s/versions/%s/files?include_unpublished=%s",
this.properties.getSubject(), this.properties.getRepo(), "example", "1.1.0.RELEASE", 1)))
.andRespond(withStatus(HttpStatus.NOT_FOUND));
setupGetPackageFiles(1, "all-package-files.json");
setupGetPackageFiles(0, "published-files.json");
setupGetPackageFiles(0, "no-package-files.json");
setupGetPackageFiles(0, "some-package-files.json");
setupGetPackageFiles(0, "all-package-files.json");
assertThat(this.service.isDistributionComplete(getReleaseInfo())).isTrue();
Set<String> digests = new LinkedHashSet<>();
digests.add("602e20176706d3cc7535f01ffdbe91b270ae5012");
digests.add("602e20176706d3cc7535f01ffdbe91b270ae5013");
digests.add("602e20176706d3cc7535f01ffdbe91b270ae5014");
assertThat(this.service.isDistributionComplete(getReleaseInfo(), digests, Duration.ofMinutes(1), Duration.ZERO))
.isTrue();
this.server.verify();
}

15
ci/images/releasescripts/src/test/java/io/spring/concourse/releasescripts/command/DistributeCommandTests.java
View File

@@ -16,6 +16,8 @@
package io.spring.concourse.releasescripts.command;
import java.util.Set;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.spring.concourse.releasescripts.ReleaseInfo;
@@ -54,7 +56,7 @@ class DistributeCommandTests {
void setup() {
MockitoAnnotations.initMocks(this);
this.objectMapper = new ObjectMapper().disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
this.command = new DistributeCommand(this.service, objectMapper);
this.command = new DistributeCommand(this.service, this.objectMapper);
}
@Test
@@ -76,15 +78,20 @@ class DistributeCommandTests {
}
@Test
@SuppressWarnings("unchecked")
void distributeWhenReleaseTypeReleaseShouldCallService() throws Exception {
ArgumentCaptor<ReleaseInfo> captor = ArgumentCaptor.forClass(ReleaseInfo.class);
ArgumentCaptor<ReleaseInfo> releaseInfoCaptor = ArgumentCaptor.forClass(ReleaseInfo.class);
ArgumentCaptor<Set<String>> artifactDigestCaptor = ArgumentCaptor.forClass(Set.class);
this.command.run(new DefaultApplicationArguments("distribute", "RELEASE", getBuildInfoLocation()));
verify(this.service).distribute(eq(ReleaseType.RELEASE.getRepo()), captor.capture());
ReleaseInfo releaseInfo = captor.getValue();
verify(this.service).distribute(eq(ReleaseType.RELEASE.getRepo()), releaseInfoCaptor.capture(),
artifactDigestCaptor.capture());
ReleaseInfo releaseInfo = releaseInfoCaptor.getValue();
assertThat(releaseInfo.getBuildName()).isEqualTo("example");
assertThat(releaseInfo.getBuildNumber()).isEqualTo("example-build-1");
assertThat(releaseInfo.getGroupId()).isEqualTo("org.example.demo");
assertThat(releaseInfo.getVersion()).isEqualTo("2.2.0");
Set<String> artifactDigests = artifactDigestCaptor.getValue();
assertThat(artifactDigests).containsExactly("aaaaaaaaa85f5c5093721f3ed0edda8ff8290yyyyyyyyyy");
}
private String getBuildInfoLocation() throws Exception {

6
ci/images/releasescripts/src/test/resources/io/spring/concourse/releasescripts/bintray/all-package-files.json
View File

@@ -8,7 +8,7 @@
"owner": "jfrog",
"created": "ISO8601 (yyyy-MM-dd'T'HH:mm:ss.SSSZ)",
"size": 1234,
"sha1": "602e20176706d3cc7535f01ffdbe91b270ae5012"
"sha256": "602e20176706d3cc7535f01ffdbe91b270ae5012"
},
{
"name": "nutcracker-1.1.pom",
@@ -19,7 +19,7 @@
"owner": "jfrog",
"created": "ISO8601 (yyyy-MM-dd'T'HH:mm:ss.SSSZ)",
"size": 1234,
"sha1": "602e20176706d3cc7535f01ffdbe91b270ae5012"
"sha256": "602e20176706d3cc7535f01ffdbe91b270ae5013"
},
{
"name": "nutcracker-1.1.jar",
@@ -30,6 +30,6 @@
"owner": "jfrog",
"created": "ISO8601 (yyyy-MM-dd'T'HH:mm:ss.SSSZ)",
"size": 1234,
"sha1": "602e20176706d3cc7535f01ffdbe91b270ae5012"
"sha256": "602e20176706d3cc7535f01ffdbe91b270ae5014"
}
]

1
ci/images/releasescripts/src/test/resources/io/spring/concourse/releasescripts/bintray/no-package-files.json Normal file
View File

@@ -0,0 +1 @@
[]

2
ci/images/releasescripts/src/test/resources/io/spring/concourse/releasescripts/bintray/published-files.json → ci/images/releasescripts/src/test/resources/io/spring/concourse/releasescripts/bintray/some-package-files.json
View File

@@ -8,6 +8,6 @@
"owner": "jfrog",
"created": "ISO8601 (yyyy-MM-dd'T'HH:mm:ss.SSSZ)",
"size": 1234,
"sha1": "602e20176706d3cc7535f01ffdbe91b270ae5012"
"sha256": "602e20176706d3cc7535f01ffdbe91b270ae5012"
}
]