Commit ce011ca3 authored by Phillip Webb's avatar Phillip Webb

Check if promotion has already occurred

Update the release tooling to check for bintray published artifacts
using SHA256 digests and to also check before attempting a promote.

See gh-21474
parent d650c5fd
......@@ -17,6 +17,8 @@
package io.spring.concourse.releasescripts.artifactory;
import java.net.URI;
import java.time.Duration;
import java.util.Set;
import io.spring.concourse.releasescripts.ReleaseInfo;
import io.spring.concourse.releasescripts.artifactory.payload.BuildInfoResponse;
......@@ -113,8 +115,12 @@ public class ArtifactoryService {
* Deploy builds from Artifactory to Bintray.
* @param sourceRepo the source repo in Artifactory.
* @param releaseInfo the resease info
* @param artifactDigests the artifact digests
*/
public void distribute(String sourceRepo, ReleaseInfo releaseInfo) {
public void distribute(String sourceRepo, ReleaseInfo releaseInfo, Set<String> artifactDigests) {
if (this.bintrayService.isDistributionComplete(releaseInfo, artifactDigests, Duration.ofMinutes(2))) {
console.log("Distribution already complete");
}
DistributionRequest request = new DistributionRequest(new String[] { sourceRepo });
RequestEntity<DistributionRequest> requestEntity = RequestEntity
.post(URI.create(DISTRIBUTION_URL + releaseInfo.getBuildName() + "/" + releaseInfo.getBuildNumber()))
......@@ -126,7 +132,7 @@ public class ArtifactoryService {
console.log("Failed to distribute.");
throw ex;
}
if (!this.bintrayService.isDistributionComplete(releaseInfo)) {
if (!this.bintrayService.isDistributionComplete(releaseInfo, artifactDigests, Duration.ofMinutes(60))) {
throw new DistributionTimeoutException("Distribution timed out.");
}
......
......@@ -16,6 +16,11 @@
package io.spring.concourse.releasescripts.artifactory.payload;
import java.util.Arrays;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
/**
* Represents the response from Artifactory's buildInfo endpoint.
*
......@@ -54,7 +59,7 @@ public class BuildInfoResponse {
}
public String getName() {
return name;
return this.name;
}
public void setName(String name) {
......@@ -83,6 +88,14 @@ public class BuildInfoResponse {
public void setVersion(String version) {
this.version = version;
}
public Set<String> getArtifactDigests() {
return Arrays.stream(this.modules).flatMap((module) -> {
Artifact[] artifacts = module.getArtifacts();
return (artifacts != null) ? Arrays.stream(artifacts) : Stream.empty();
}).map(Artifact::getSha256).collect(Collectors.toSet());
}
}
......@@ -105,6 +118,8 @@ public class BuildInfoResponse {
private String id;
private Artifact[] artifacts;
public String getId() {
return this.id;
}
......@@ -113,6 +128,38 @@ public class BuildInfoResponse {
this.id = id;
}
public Artifact[] getArtifacts() {
return this.artifacts;
}
public void setArtifacts(Artifact[] artifacts) {
this.artifacts = artifacts;
}
}
public static class Artifact {
private String name;
private String sha256;
public String getName() {
return this.name;
}
public void setName(String name) {
this.name = name;
}
public String getSha256() {
return this.sha256;
}
public void setSha256(String sha256) {
this.sha256 = sha256;
}
}
}
......@@ -17,8 +17,9 @@
package io.spring.concourse.releasescripts.bintray;
import java.net.URI;
import java.util.Objects;
import java.util.concurrent.TimeUnit;
import java.time.Duration;
import java.util.HashSet;
import java.util.Set;
import io.spring.concourse.releasescripts.ReleaseInfo;
import io.spring.concourse.releasescripts.sonatype.SonatypeProperties;
......@@ -27,7 +28,6 @@ import io.spring.concourse.releasescripts.system.ConsoleLogger;
import org.awaitility.core.ConditionTimeoutException;
import org.springframework.boot.web.client.RestTemplateBuilder;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.http.RequestEntity;
import org.springframework.stereotype.Component;
......@@ -72,24 +72,17 @@ public class BintrayService {
this.restTemplate = builder.build();
}
public boolean isDistributionComplete(ReleaseInfo releaseInfo) {
RequestEntity<Void> allFilesRequest = getRequest(releaseInfo, 1);
Object[] allFiles = waitAtMost(5, TimeUnit.MINUTES).with().pollDelay(20, TimeUnit.SECONDS).until(() -> {
try {
return this.restTemplate.exchange(allFilesRequest, Object[].class).getBody();
}
catch (HttpClientErrorException ex) {
if (ex.getStatusCode() != HttpStatus.NOT_FOUND) {
throw ex;
}
return null;
}
}, Objects::nonNull);
RequestEntity<Void> publishedFilesRequest = getRequest(releaseInfo, 0);
public boolean isDistributionComplete(ReleaseInfo releaseInfo, Set<String> requiredDigets, Duration timeout) {
return isDistributionComplete(releaseInfo, requiredDigets, timeout, Duration.ofSeconds(20));
}
public boolean isDistributionComplete(ReleaseInfo releaseInfo, Set<String> requiredDigets, Duration timeout,
Duration pollDelay) {
RequestEntity<Void> request = getRequest(releaseInfo, 0);
try {
waitAtMost(120, TimeUnit.MINUTES).with().pollDelay(20, TimeUnit.SECONDS).until(() -> {
Object[] publishedFiles = this.restTemplate.exchange(publishedFilesRequest, Object[].class).getBody();
return allFiles.length == publishedFiles.length;
waitAtMost(timeout).with().pollDelay(pollDelay).until(() -> {
PackageFile[] published = this.restTemplate.exchange(request, PackageFile[].class).getBody();
return hasPublishedAll(published, requiredDigets);
});
}
catch (ConditionTimeoutException ex) {
......@@ -98,6 +91,17 @@ public class BintrayService {
return true;
}
private boolean hasPublishedAll(PackageFile[] published, Set<String> requiredDigets) {
if (published == null || published.length == 0) {
return false;
}
Set<String> remaining = new HashSet<>(requiredDigets);
for (PackageFile publishedFile : published) {
remaining.remove(publishedFile.getSha256());
}
return remaining.isEmpty();
}
private RequestEntity<Void> getRequest(ReleaseInfo releaseInfo, int includeUnpublished) {
return RequestEntity.get(URI.create(BINTRAY_URL + "packages/" + this.bintrayProperties.getSubject() + "/"
+ this.bintrayProperties.getRepo() + "/" + releaseInfo.getGroupId() + "/versions/"
......
/*
* Copyright 2020 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.spring.concourse.releasescripts.bintray;
/**
* Details for a single packaged file.
*
* @author Phillip Webb
*/
public class PackageFile {
private String name;
private String sha256;
public String getName() {
return this.name;
}
public String getSha256() {
return this.sha256;
}
}
......@@ -19,12 +19,14 @@ package io.spring.concourse.releasescripts.command;
import java.io.File;
import java.nio.file.Files;
import java.util.List;
import java.util.Set;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.spring.concourse.releasescripts.ReleaseInfo;
import io.spring.concourse.releasescripts.ReleaseType;
import io.spring.concourse.releasescripts.artifactory.ArtifactoryService;
import io.spring.concourse.releasescripts.artifactory.payload.BuildInfoResponse;
import io.spring.concourse.releasescripts.artifactory.payload.BuildInfoResponse.BuildInfo;
import org.springframework.boot.ApplicationArguments;
import org.springframework.stereotype.Component;
......@@ -38,12 +40,12 @@ import org.springframework.util.Assert;
@Component
public class DistributeCommand implements Command {
private final ArtifactoryService service;
private final ArtifactoryService artifactoryService;
private final ObjectMapper objectMapper;
public DistributeCommand(ArtifactoryService service, ObjectMapper objectMapper) {
this.service = service;
public DistributeCommand(ArtifactoryService artifactoryService, ObjectMapper objectMapper) {
this.artifactoryService = artifactoryService;
this.objectMapper = objectMapper;
}
......@@ -60,8 +62,10 @@ public class DistributeCommand implements Command {
String buildInfoLocation = nonOptionArgs.get(2);
byte[] content = Files.readAllBytes(new File(buildInfoLocation).toPath());
BuildInfoResponse buildInfoResponse = this.objectMapper.readValue(content, BuildInfoResponse.class);
ReleaseInfo releaseInfo = ReleaseInfo.from(buildInfoResponse.getBuildInfo());
this.service.distribute(type.getRepo(), releaseInfo);
BuildInfo buildInfo = buildInfoResponse.getBuildInfo();
Set<String> artifactDigests = buildInfo.getArtifactDigests();
ReleaseInfo releaseInfo = ReleaseInfo.from(buildInfo);
this.artifactoryService.distribute(type.getRepo(), releaseInfo, artifactDigests);
}
}
......@@ -16,10 +16,15 @@
package io.spring.concourse.releasescripts.artifactory;
import java.time.Duration;
import java.util.Collections;
import java.util.Set;
import io.spring.concourse.releasescripts.ReleaseInfo;
import io.spring.concourse.releasescripts.bintray.BintrayService;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import org.mockito.InOrder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
......@@ -35,10 +40,13 @@ import org.springframework.util.Base64Utils;
import org.springframework.web.client.HttpClientErrorException;
import static org.assertj.core.api.Assertions.assertThatExceptionOfType;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.BDDMockito.given;
import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.verifyNoInteractions;
import static org.mockito.Mockito.verifyNoMoreInteractions;
import static org.springframework.test.web.client.match.MockRestRequestMatchers.content;
import static org.springframework.test.web.client.match.MockRestRequestMatchers.header;
import static org.springframework.test.web.client.match.MockRestRequestMatchers.method;
......@@ -55,6 +63,10 @@ import static org.springframework.test.web.client.response.MockRestResponseCreat
@EnableConfigurationProperties(ArtifactoryProperties.class)
class ArtifactoryServiceTests {
private static final Duration SHORT_TIMEOUT = Duration.ofMinutes(2);
private static final Duration LONG_TIMEOUT = Duration.ofMinutes(60);
@Autowired
private ArtifactoryService service;
......@@ -118,9 +130,10 @@ class ArtifactoryServiceTests {
}
@Test
@SuppressWarnings("unchecked")
void distributeWhenSuccessful() throws Exception {
ReleaseInfo releaseInfo = getReleaseInfo();
given(this.bintrayService.isDistributionComplete(releaseInfo)).willReturn(true);
given(this.bintrayService.isDistributionComplete(eq(releaseInfo), (Set<String>) any(), any())).willReturn(true);
this.server.expect(requestTo("https://repo.spring.io/api/build/distribute/example-build/example-build-1"))
.andExpect(method(HttpMethod.POST))
.andExpect(content().json(
......@@ -128,9 +141,12 @@ class ArtifactoryServiceTests {
.andExpect(header("Authorization", "Basic " + Base64Utils.encodeToString(String
.format("%s:%s", this.properties.getUsername(), this.properties.getPassword()).getBytes())))
.andExpect(header("Content-Type", MediaType.APPLICATION_JSON.toString())).andRespond(withSuccess());
this.service.distribute("libs-release-local", releaseInfo);
Set<String> artifactDigests = Collections.singleton("602e20176706d3cc7535f01ffdbe91b270ae5014");
this.service.distribute("libs-release-local", releaseInfo, artifactDigests);
this.server.verify();
verify(this.bintrayService, times(1)).isDistributionComplete(releaseInfo);
InOrder ordered = inOrder(this.bintrayService);
ordered.verify(this.bintrayService).isDistributionComplete(releaseInfo, artifactDigests, SHORT_TIMEOUT);
ordered.verify(this.bintrayService).isDistributionComplete(releaseInfo, artifactDigests, LONG_TIMEOUT);
}
@Test
......@@ -144,16 +160,22 @@ class ArtifactoryServiceTests {
.format("%s:%s", this.properties.getUsername(), this.properties.getPassword()).getBytes())))
.andExpect(header("Content-Type", MediaType.APPLICATION_JSON.toString()))
.andRespond(withStatus(HttpStatus.FORBIDDEN));
Set<String> artifactDigests = Collections.singleton("602e20176706d3cc7535f01ffdbe91b270ae5014");
assertThatExceptionOfType(HttpClientErrorException.class)
.isThrownBy(() -> this.service.distribute("libs-release-local", releaseInfo));
.isThrownBy(() -> this.service.distribute("libs-release-local", releaseInfo, artifactDigests));
this.server.verify();
verifyNoInteractions(this.bintrayService);
verify(this.bintrayService, times(1)).isDistributionComplete(releaseInfo, artifactDigests, SHORT_TIMEOUT);
verifyNoMoreInteractions(this.bintrayService);
}
@Test
@SuppressWarnings("unchecked")
void distributeWhenGettingPackagesTimesOut() throws Exception {
ReleaseInfo releaseInfo = getReleaseInfo();
given(this.bintrayService.isDistributionComplete(releaseInfo)).willReturn(false);
given(this.bintrayService.isDistributionComplete(eq(releaseInfo), (Set<String>) any(), any()))
.willReturn(false);
given(this.bintrayService.isDistributionComplete(eq(releaseInfo), (Set<String>) any(), any()))
.willReturn(false);
this.server.expect(requestTo("https://repo.spring.io/api/build/distribute/example-build/example-build-1"))
.andExpect(method(HttpMethod.POST))
.andExpect(content().json(
......@@ -161,10 +183,13 @@ class ArtifactoryServiceTests {
.andExpect(header("Authorization", "Basic " + Base64Utils.encodeToString(String
.format("%s:%s", this.properties.getUsername(), this.properties.getPassword()).getBytes())))
.andExpect(header("Content-Type", MediaType.APPLICATION_JSON.toString())).andRespond(withSuccess());
Set<String> artifactDigests = Collections.singleton("602e20176706d3cc7535f01ffdbe91b270ae5014");
assertThatExceptionOfType(DistributionTimeoutException.class)
.isThrownBy(() -> this.service.distribute("libs-release-local", releaseInfo));
.isThrownBy(() -> this.service.distribute("libs-release-local", releaseInfo, artifactDigests));
this.server.verify();
verify(this.bintrayService, times(1)).isDistributionComplete(releaseInfo);
InOrder ordered = inOrder(this.bintrayService);
ordered.verify(this.bintrayService).isDistributionComplete(releaseInfo, artifactDigests, SHORT_TIMEOUT);
ordered.verify(this.bintrayService).isDistributionComplete(releaseInfo, artifactDigests, LONG_TIMEOUT);
}
private ReleaseInfo getReleaseInfo() {
......
......@@ -16,6 +16,10 @@
package io.spring.concourse.releasescripts.bintray;
import java.time.Duration;
import java.util.LinkedHashSet;
import java.util.Set;
import io.spring.concourse.releasescripts.ReleaseInfo;
import io.spring.concourse.releasescripts.sonatype.SonatypeProperties;
import io.spring.concourse.releasescripts.sonatype.SonatypeService;
......@@ -28,7 +32,6 @@ import org.springframework.boot.test.autoconfigure.web.client.RestClientTest;
import org.springframework.boot.test.mock.mockito.MockBean;
import org.springframework.core.io.ClassPathResource;
import org.springframework.http.HttpMethod;
import org.springframework.http.HttpStatus;
import org.springframework.http.MediaType;
import org.springframework.test.web.client.ExpectedCount;
import org.springframework.test.web.client.MockRestServiceServer;
......@@ -41,7 +44,6 @@ import static org.springframework.test.web.client.match.MockRestRequestMatchers.
import static org.springframework.test.web.client.match.MockRestRequestMatchers.header;
import static org.springframework.test.web.client.match.MockRestRequestMatchers.method;
import static org.springframework.test.web.client.match.MockRestRequestMatchers.requestTo;
import static org.springframework.test.web.client.response.MockRestResponseCreators.withStatus;
import static org.springframework.test.web.client.response.MockRestResponseCreators.withSuccess;
/**
......@@ -75,15 +77,15 @@ class BintrayServiceTests {
@Test
void isDistributionComplete() throws Exception {
this.server
.expect(requestTo(String.format(
"https://api.bintray.com/packages/%s/%s/%s/versions/%s/files?include_unpublished=%s",
this.properties.getSubject(), this.properties.getRepo(), "example", "1.1.0.RELEASE", 1)))
.andRespond(withStatus(HttpStatus.NOT_FOUND));
setupGetPackageFiles(1, "all-package-files.json");
setupGetPackageFiles(0, "published-files.json");
setupGetPackageFiles(0, "no-package-files.json");
setupGetPackageFiles(0, "some-package-files.json");
setupGetPackageFiles(0, "all-package-files.json");
assertThat(this.service.isDistributionComplete(getReleaseInfo())).isTrue();
Set<String> digests = new LinkedHashSet<>();
digests.add("602e20176706d3cc7535f01ffdbe91b270ae5012");
digests.add("602e20176706d3cc7535f01ffdbe91b270ae5013");
digests.add("602e20176706d3cc7535f01ffdbe91b270ae5014");
assertThat(this.service.isDistributionComplete(getReleaseInfo(), digests, Duration.ofMinutes(1), Duration.ZERO))
.isTrue();
this.server.verify();
}
......
......@@ -16,6 +16,8 @@
package io.spring.concourse.releasescripts.command;
import java.util.Set;
import com.fasterxml.jackson.databind.DeserializationFeature;
import com.fasterxml.jackson.databind.ObjectMapper;
import io.spring.concourse.releasescripts.ReleaseInfo;
......@@ -54,7 +56,7 @@ class DistributeCommandTests {
void setup() {
MockitoAnnotations.initMocks(this);
this.objectMapper = new ObjectMapper().disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES);
this.command = new DistributeCommand(this.service, objectMapper);
this.command = new DistributeCommand(this.service, this.objectMapper);
}
@Test
......@@ -76,15 +78,20 @@ class DistributeCommandTests {
}
@Test
@SuppressWarnings("unchecked")
void distributeWhenReleaseTypeReleaseShouldCallService() throws Exception {
ArgumentCaptor<ReleaseInfo> captor = ArgumentCaptor.forClass(ReleaseInfo.class);
ArgumentCaptor<ReleaseInfo> releaseInfoCaptor = ArgumentCaptor.forClass(ReleaseInfo.class);
ArgumentCaptor<Set<String>> artifactDigestCaptor = ArgumentCaptor.forClass(Set.class);
this.command.run(new DefaultApplicationArguments("distribute", "RELEASE", getBuildInfoLocation()));
verify(this.service).distribute(eq(ReleaseType.RELEASE.getRepo()), captor.capture());
ReleaseInfo releaseInfo = captor.getValue();
verify(this.service).distribute(eq(ReleaseType.RELEASE.getRepo()), releaseInfoCaptor.capture(),
artifactDigestCaptor.capture());
ReleaseInfo releaseInfo = releaseInfoCaptor.getValue();
assertThat(releaseInfo.getBuildName()).isEqualTo("example");
assertThat(releaseInfo.getBuildNumber()).isEqualTo("example-build-1");
assertThat(releaseInfo.getGroupId()).isEqualTo("org.example.demo");
assertThat(releaseInfo.getVersion()).isEqualTo("2.2.0");
Set<String> artifactDigests = artifactDigestCaptor.getValue();
assertThat(artifactDigests).containsExactly("aaaaaaaaa85f5c5093721f3ed0edda8ff8290yyyyyyyyyy");
}
private String getBuildInfoLocation() throws Exception {
......
......@@ -8,7 +8,7 @@
"owner": "jfrog",
"created": "ISO8601 (yyyy-MM-dd'T'HH:mm:ss.SSSZ)",
"size": 1234,
"sha1": "602e20176706d3cc7535f01ffdbe91b270ae5012"
"sha256": "602e20176706d3cc7535f01ffdbe91b270ae5012"
},
{
"name": "nutcracker-1.1.pom",
......@@ -19,7 +19,7 @@
"owner": "jfrog",
"created": "ISO8601 (yyyy-MM-dd'T'HH:mm:ss.SSSZ)",
"size": 1234,
"sha1": "602e20176706d3cc7535f01ffdbe91b270ae5012"
"sha256": "602e20176706d3cc7535f01ffdbe91b270ae5013"
},
{
"name": "nutcracker-1.1.jar",
......@@ -30,6 +30,6 @@
"owner": "jfrog",
"created": "ISO8601 (yyyy-MM-dd'T'HH:mm:ss.SSSZ)",
"size": 1234,
"sha1": "602e20176706d3cc7535f01ffdbe91b270ae5012"
"sha256": "602e20176706d3cc7535f01ffdbe91b270ae5014"
}
]
\ No newline at end of file
......@@ -8,6 +8,6 @@
"owner": "jfrog",
"created": "ISO8601 (yyyy-MM-dd'T'HH:mm:ss.SSSZ)",
"size": 1234,
"sha1": "602e20176706d3cc7535f01ffdbe91b270ae5012"
"sha256": "602e20176706d3cc7535f01ffdbe91b270ae5012"
}
]
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment