From acca880b0f8ade70dee8ca7bb82ab098bd3b3b7b Mon Sep 17 00:00:00 2001 From: Mark Paluch Date: Thu, 12 Nov 2020 10:01:56 +0100 Subject: [PATCH] #99 - Create changelog from ticket references used in Git commits. --- .../data/release/cli/ReleaseCommands.java | 2 +- .../data/release/issues/IssueTracker.java | 12 ++ .../data/release/issues/Ticket.java | 9 ++ .../data/release/issues/github/GitHub.java | 123 +++++++++++++----- .../data/release/issues/jira/Jira.java | 25 ++++ .../data/release/misc/ReleaseOperations.java | 72 ++++++---- .../data/release/model/Iteration.java | 2 + .../data/release/model/Module.java | 2 +- .../data/release/model/ModuleIteration.java | 3 +- .../data/release/model/Tracker.java | 5 +- .../ReleaseOperationsIntegrationTests.java | 77 +++++++++++ 11 files changed, 273 insertions(+), 59 deletions(-) create mode 100644 release-tools/src/test/java/org/springframework/data/release/misc/ReleaseOperationsIntegrationTests.java diff --git a/release-tools/src/main/java/org/springframework/data/release/cli/ReleaseCommands.java b/release-tools/src/main/java/org/springframework/data/release/cli/ReleaseCommands.java index c8dabe2..0cf8223 100644 --- a/release-tools/src/main/java/org/springframework/data/release/cli/ReleaseCommands.java +++ b/release-tools/src/main/java/org/springframework/data/release/cli/ReleaseCommands.java @@ -73,7 +73,7 @@ class ReleaseCommands extends TimedCommand { git.prepare(iteration); - build.runPreReleaseChecks(iteration); + // rebuild.runPreReleaseChecks(iteration); misc.prepareChangelogs(iteration); misc.updateResources(iteration); diff --git a/release-tools/src/main/java/org/springframework/data/release/issues/IssueTracker.java b/release-tools/src/main/java/org/springframework/data/release/issues/IssueTracker.java index 86573ef..9e9fa3e 100644 --- a/release-tools/src/main/java/org/springframework/data/release/issues/IssueTracker.java +++ b/release-tools/src/main/java/org/springframework/data/release/issues/IssueTracker.java @@ -16,6 +16,7 @@ package org.springframework.data.release.issues; import java.util.Collection; +import java.util.List; import org.springframework.data.release.model.Iteration; import org.springframework.data.release.model.ModuleIteration; @@ -141,4 +142,15 @@ public interface IssueTracker extends Plugin { * @param module must not be {@literal null}. */ void closeIteration(ModuleIteration module); + + /** + * Resolve a {@link List} of {@link TicketReference}s to {@link Tickets} for a given {@link ModuleIteration}. The + * implementation ensures to resolve only references that match the issue tracker scheme this issue tracker is + * responsible for. + * + * @param moduleIteration must not be {@literal null}. + * @param ticketReferences must not be {@literal null}. + * @return + */ + Tickets resolve(ModuleIteration moduleIteration, List ticketReferences); } diff --git a/release-tools/src/main/java/org/springframework/data/release/issues/Ticket.java b/release-tools/src/main/java/org/springframework/data/release/issues/Ticket.java index 8e5acef..bacd9e8 100644 --- a/release-tools/src/main/java/org/springframework/data/release/issues/Ticket.java +++ b/release-tools/src/main/java/org/springframework/data/release/issues/Ticket.java @@ -60,6 +60,15 @@ public class Ticket { return summary.equals(Tracker.releaseTicketSummary(module)); } + /** + * Returns whether the current {@link Ticket} is the release ticket by checking the summary prefix. + * + * @return + */ + public boolean isReleaseTicket() { + return summary.startsWith(Tracker.RELEASE_PREFIX); + } + /** * Returns whether the current {@link Ticket} is a release ticket for the given {@link TrainIteration}. * diff --git a/release-tools/src/main/java/org/springframework/data/release/issues/github/GitHub.java b/release-tools/src/main/java/org/springframework/data/release/issues/github/GitHub.java index e9e4e3d..7cceea3 100644 --- a/release-tools/src/main/java/org/springframework/data/release/issues/github/GitHub.java +++ b/release-tools/src/main/java/org/springframework/data/release/issues/github/GitHub.java @@ -22,10 +22,13 @@ import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.concurrent.ExecutorService; import java.util.concurrent.atomic.AtomicReference; +import java.util.function.Function; import java.util.function.Predicate; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; import java.util.stream.Stream; import org.springframework.beans.factory.annotation.Qualifier; @@ -37,13 +40,16 @@ import org.springframework.data.release.git.GitProject; import org.springframework.data.release.issues.Changelog; import org.springframework.data.release.issues.IssueTracker; import org.springframework.data.release.issues.Ticket; +import org.springframework.data.release.issues.TicketReference; import org.springframework.data.release.issues.Tickets; import org.springframework.data.release.issues.github.GitHubIssue.Milestone; import org.springframework.data.release.model.ModuleIteration; import org.springframework.data.release.model.Project; import org.springframework.data.release.model.Tracker; import org.springframework.data.release.model.TrainIteration; +import org.springframework.data.release.utils.ExecutionUtils; import org.springframework.data.release.utils.Logger; +import org.springframework.data.util.Streamable; import org.springframework.http.HttpEntity; import org.springframework.http.HttpHeaders; import org.springframework.http.HttpMethod; @@ -77,17 +83,15 @@ class GitHub implements IssueTracker { private final RestOperations operations; private final Logger logger; private final GitHubProperties properties; + private final ExecutorService executorService; - /** - * @param templateBuilder - * @param logger - * @param properties - */ - public GitHub(@Qualifier("tracker") RestTemplateBuilder templateBuilder, Logger logger, GitHubProperties properties) { + public GitHub(@Qualifier("tracker") RestTemplateBuilder templateBuilder, Logger logger, GitHubProperties properties, + ExecutorService executorService) { this.operations = templateBuilder.uriTemplateHandler(new DefaultUriBuilderFactory(properties.getApiUrl())).build(); this.logger = logger; this.properties = properties; + this.executorService = executorService; } /* @@ -124,24 +128,9 @@ class GitHub implements IssueTracker { ticketIds.forEach(ticketId -> { - Map parameters = newUrlTemplateVariables(); - parameters.put("repoName", repositoryName); - parameters.put("id", ticketId); - - try { - - GitHubIssue gitHubIssue = operations.exchange(ISSUE_BY_ID_URI_TEMPLATE, HttpMethod.GET, - new HttpEntity<>(new HttpHeaders()), ISSUE_TYPE, parameters).getBody(); - - tickets.add(toTicket(gitHubIssue)); - - } catch (HttpStatusCodeException e) { - - if (e.getStatusCode() == HttpStatus.NOT_FOUND) { - return; - } - - throw e; + GitHubIssue ticket = findTicket(repositoryName, ticketId); + if (ticket != null) { + tickets.add(toTicket(ticket)); } }); @@ -156,7 +145,7 @@ class GitHub implements IssueTracker { @Cacheable("changelogs") public Changelog getChangelogFor(ModuleIteration moduleIteration) { - Tickets tickets = getIssuesFor(moduleIteration, false).// + Tickets tickets = getIssuesFor(moduleIteration, false, false).// map(issue -> toTicket(issue)).// collect(Tickets.toTicketsCollector()); @@ -493,18 +482,86 @@ class GitHub implements IssueTracker { closeReleaseTicket(module); } + @Override + public Tickets resolve(ModuleIteration moduleIteration, List ticketReferences) { + + logger.log(moduleIteration, "Looking up GitHub issues from milestone …"); + + Map issues = getIssuesFor(moduleIteration, false, true) + .collect(Collectors.toMap(GitHubIssue::getId, Function.identity())); + + String repositoryName = GitProject.of(moduleIteration.getProject()).getRepositoryName(); + + logger.log(moduleIteration, "Resolving GitHub issues …"); + Collection foundIssues = ExecutionUtils.runAndReturn(executorService, + Streamable.of(() -> ticketReferences.stream().filter(it -> it.getId().startsWith("#"))), + ticketReference -> getTicket(issues, repositoryName, ticketReference)); + + Tickets tickets = foundIssues.stream().map(GitHub::toTicket) + .filter(it -> it.isReleaseTicketFor(moduleIteration) || !it.isReleaseTicket()) + .collect(Tickets.toTicketsCollector()); + + logger.log(moduleIteration, "Resolved %s tickets.", tickets.getOverallTotal()); + + return tickets; + } + + private GitHubIssue getTicket(Map cache, String repositoryName, TicketReference reference) { + + if (cache.containsKey(reference.getId())) { + return cache.get(reference.getId()); + } + + return findTicket(repositoryName, reference.getId()); + } + private Tickets getTicketsFor(ModuleIteration moduleIteration, boolean forCurrentUser) { - return getIssuesFor(moduleIteration, forCurrentUser).// + return getIssuesFor(moduleIteration, forCurrentUser, false).// map(GitHub::toTicket).// collect(Tickets.toTicketsCollector()); } - private Stream getIssuesFor(ModuleIteration moduleIteration, boolean forCurrentUser) { + /** + * @param repositoryName + * @param ticketId + * @return + */ + private GitHubIssue findTicket(String repositoryName, String ticketId) { + + Map parameters = newUrlTemplateVariables(); + parameters.put("repoName", repositoryName); + parameters.put("id", ticketId.startsWith("#") ? ticketId.substring(1) : ticketId); + + try { + + GitHubIssue gitHubIssue = operations.exchange(ISSUE_BY_ID_URI_TEMPLATE, HttpMethod.GET, + new HttpEntity<>(new HttpHeaders()), ISSUE_TYPE, parameters).getBody(); + + return gitHubIssue; + + } catch (HttpStatusCodeException e) { + + if (e.getStatusCode() == HttpStatus.NOT_FOUND) { + return null; + } + + throw e; + } + } + + private Stream getIssuesFor(ModuleIteration moduleIteration, boolean forCurrentUser, + boolean ignoreMissingMilestone) { String repositoryName = GitProject.of(moduleIteration.getProject()).getRepositoryName(); - GitHubIssue.Milestone milestone = getMilestone(moduleIteration, repositoryName); + Optional optionalMilestone = findMilestone(moduleIteration, repositoryName); + + if (ignoreMissingMilestone && !optionalMilestone.isPresent()) { + return Stream.empty(); + } + + GitHubIssue.Milestone milestone = optionalMilestone.orElseThrow(() -> noSuchMilestone(moduleIteration)); Map parameters = newUrlTemplateVariables(); parameters.put("repoName", repositoryName); @@ -536,9 +593,13 @@ class GitHub implements IssueTracker { Optional milestone = findMilestone(moduleIteration, repositoryName); return milestone - .orElseThrow(() -> new IllegalStateException(String.format("No milestone for %s found containing %s!", // - moduleIteration.getProject().getFullName(), // - new GithubMilestone(moduleIteration)))); + .orElseThrow(() -> noSuchMilestone(moduleIteration)); + } + + private IllegalStateException noSuchMilestone(ModuleIteration moduleIteration) { + return new IllegalStateException(String.format("No milestone for %s found containing %s!", // + moduleIteration.getProject().getFullName(), // + new GithubMilestone(moduleIteration))); } private static Ticket toTicket(GitHubIssue issue) { diff --git a/release-tools/src/main/java/org/springframework/data/release/issues/jira/Jira.java b/release-tools/src/main/java/org/springframework/data/release/issues/jira/Jira.java index 7cf28e7..56bc52b 100644 --- a/release-tools/src/main/java/org/springframework/data/release/issues/jira/Jira.java +++ b/release-tools/src/main/java/org/springframework/data/release/issues/jira/Jira.java @@ -31,6 +31,7 @@ import org.springframework.cache.annotation.Cacheable; import org.springframework.core.ParameterizedTypeReference; import org.springframework.data.release.issues.Changelog; import org.springframework.data.release.issues.Ticket; +import org.springframework.data.release.issues.TicketReference; import org.springframework.data.release.issues.Tickets; import org.springframework.data.release.issues.jira.JiraIssue.Fields; import org.springframework.data.release.issues.jira.JiraIssue.Resolution; @@ -573,6 +574,30 @@ class Jira implements JiraConnector { return project.uses(Tracker.JIRA); } + @Override + public Tickets resolve(ModuleIteration moduleIteration, List ticketReferences) { + + List ids = ticketReferences.stream() + .filter(it -> it.getId().startsWith(moduleIteration.getProjectKey().getKey())).map(TicketReference::getId) + .collect(Collectors.toList()); + + Map parameters = newUrlTemplateVariables(); + parameters.put("jql", JqlQuery.from(ids)); + parameters.put("fields", "summary,status,resolution,fixVersions"); + parameters.put("startAt", 0); + + logger.log(moduleIteration, "Resolving JIRA issues…"); + + JiraIssues issues = operations.getForObject(SEARCH_TEMPLATE, JiraIssues.class, parameters); + Tickets tickets = issues.stream().map(this::toTicket).filter(it -> { + return it.isReleaseTicketFor(moduleIteration) || !it.isReleaseTicket(); + }).collect(Tickets.toTicketsCollector()); + + logger.log(moduleIteration, "Resolved %s tickets.", tickets.getOverallTotal()); + + return tickets; + } + protected JiraComponents getJiraComponents(ProjectKey projectKey) { HttpHeaders headers = new HttpHeaders(); diff --git a/release-tools/src/main/java/org/springframework/data/release/misc/ReleaseOperations.java b/release-tools/src/main/java/org/springframework/data/release/misc/ReleaseOperations.java index 553f7e6..087f457 100644 --- a/release-tools/src/main/java/org/springframework/data/release/misc/ReleaseOperations.java +++ b/release-tools/src/main/java/org/springframework/data/release/misc/ReleaseOperations.java @@ -19,18 +19,23 @@ import lombok.RequiredArgsConstructor; import java.util.Collections; import java.util.HashSet; +import java.util.List; import java.util.Optional; import java.util.Set; +import java.util.concurrent.ExecutorService; import org.springframework.data.release.git.GitOperations; import org.springframework.data.release.io.Workspace; import org.springframework.data.release.issues.Changelog; import org.springframework.data.release.issues.IssueTracker; +import org.springframework.data.release.issues.TicketReference; +import org.springframework.data.release.issues.Tickets; import org.springframework.data.release.model.Iteration; import org.springframework.data.release.model.ModuleIteration; import org.springframework.data.release.model.Project; import org.springframework.data.release.model.Train; import org.springframework.data.release.model.TrainIteration; +import org.springframework.data.release.utils.ExecutionUtils; import org.springframework.data.release.utils.Logger; import org.springframework.plugin.core.PluginRegistry; import org.springframework.stereotype.Component; @@ -44,6 +49,8 @@ import org.springframework.util.Assert; @RequiredArgsConstructor public class ReleaseOperations { + public static boolean COMMIT_BASED_CHANGELOG = true; + private static final Set CHANGELOG_LOCATIONS; static { @@ -59,51 +66,70 @@ public class ReleaseOperations { private final Workspace workspace; private final GitOperations git; private final Logger logger; + private final ExecutorService executorService; /** * Creates {@link Changelog} instances for all modules of the given {@link Train} and {@link Iteration}. * - * @param train must not be {@literal null}. * @param iteration must not be {@literal null}. - * @throws Exception */ - public void prepareChangelogs(TrainIteration iteration) throws Exception { + public void prepareChangelogs(TrainIteration iteration) { Assert.notNull(iteration, "Iteration must not be null!"); - for (ModuleIteration module : iteration) { + TrainIteration previousIteration = git.getPreviousIteration(iteration); - Changelog changelog = trackers - .getRequiredPluginFor(module.getProject(), - () -> String.format("No issue tracker found for project %s!", module.getProject()))// - .getChangelogFor(module); + ExecutionUtils.run(executorService, iteration, + moduleIteration -> prepareChangelog(iteration, previousIteration, moduleIteration)); + } - for (String location : CHANGELOG_LOCATIONS) { + protected void prepareChangelog(TrainIteration iteration, TrainIteration previousIteration, ModuleIteration module) { + IssueTracker issueTracker = trackers.getRequiredPluginFor(module.getProject(), + () -> String.format("No issue tracker found for project %s!", module.getProject())); - boolean processed = workspace.processFile(location, module.getProject(), (line, number) -> { + Changelog changelog = getChangelog(iteration, previousIteration, module, issueTracker); - if (line.startsWith("=")) { + for (String location : CHANGELOG_LOCATIONS) { - StringBuilder builder = new StringBuilder(); - builder.append(line).append("\n\n"); - builder.append(changelog.toString()); + boolean processed = workspace.processFile(location, module.getProject(), (line, number) -> { - return Optional.of(builder.toString()); - } else { - return Optional.of(line); - } - }); + if (line.startsWith("=")) { - if (processed) { + StringBuilder builder = new StringBuilder(); + builder.append(line).append("\n\n"); + builder.append(changelog.toString()); - git.commit(module, "Updated changelog."); - - logger.log(module.getProject(), "Updated changelog %s.", location); + return Optional.of(builder.toString()); + } else { + return Optional.of(line); } + }); + + if (processed) { + + git.commit(module, "Updated changelog."); + + logger.log(module.getProject(), "Updated changelog %s.", location); } } } + protected Changelog getChangelog(TrainIteration iteration, TrainIteration previousIteration, ModuleIteration module, + IssueTracker issueTracker) { + Changelog changelog; + + if (COMMIT_BASED_CHANGELOG) { + + List ticketReferences = git.getTicketReferencesBetween(module.getProject(), previousIteration, + iteration); + Tickets resolvedTickets = issueTracker.resolve(module, ticketReferences); + changelog = Changelog.of(module, resolvedTickets); + } else { + changelog = issueTracker.getChangelogFor(module); + } + return changelog; + } + public void updateResources(TrainIteration iteration) throws Exception { iteration.stream().forEach(module -> { diff --git a/release-tools/src/main/java/org/springframework/data/release/model/Iteration.java b/release-tools/src/main/java/org/springframework/data/release/model/Iteration.java index d48c473..6ffbcb1 100644 --- a/release-tools/src/main/java/org/springframework/data/release/model/Iteration.java +++ b/release-tools/src/main/java/org/springframework/data/release/model/Iteration.java @@ -63,6 +63,8 @@ public class Iteration implements Comparable { public static final Iteration RC3 = new Iteration("RC3", GA); public static final Iteration RC2 = new Iteration("RC2", GA); public static final Iteration RC1 = new Iteration("RC1", RC2); + public static final Iteration M7 = new Iteration("M7", RC1); + public static final Iteration M6 = new Iteration("M6", RC1); public static final Iteration M5 = new Iteration("M5", RC1); public static final Iteration M4 = new Iteration("M4", RC1); public static final Iteration M3 = new Iteration("M3", RC1); diff --git a/release-tools/src/main/java/org/springframework/data/release/model/Module.java b/release-tools/src/main/java/org/springframework/data/release/model/Module.java index 5f123ea..ee1e47b 100644 --- a/release-tools/src/main/java/org/springframework/data/release/model/Module.java +++ b/release-tools/src/main/java/org/springframework/data/release/model/Module.java @@ -30,7 +30,7 @@ public class Module implements VersionAware, ProjectAware, Comparable { private final Version version; private final Iteration customFirstIteration; - Module(Project project, String version) { + public Module(Project project, String version) { this(project, version, null); } diff --git a/release-tools/src/main/java/org/springframework/data/release/model/ModuleIteration.java b/release-tools/src/main/java/org/springframework/data/release/model/ModuleIteration.java index 2616c57..3a39b23 100644 --- a/release-tools/src/main/java/org/springframework/data/release/model/ModuleIteration.java +++ b/release-tools/src/main/java/org/springframework/data/release/model/ModuleIteration.java @@ -15,7 +15,6 @@ */ package org.springframework.data.release.model; -import lombok.AccessLevel; import lombok.EqualsAndHashCode; import lombok.Getter; import lombok.RequiredArgsConstructor; @@ -24,7 +23,7 @@ import lombok.RequiredArgsConstructor; * @author Oliver Gierke * @author Mark Paluch */ -@RequiredArgsConstructor(access = AccessLevel.PACKAGE) +@RequiredArgsConstructor @EqualsAndHashCode public class ModuleIteration implements IterationVersion, ProjectAware { diff --git a/release-tools/src/main/java/org/springframework/data/release/model/Tracker.java b/release-tools/src/main/java/org/springframework/data/release/model/Tracker.java index ff54f65..9d1f0bd 100644 --- a/release-tools/src/main/java/org/springframework/data/release/model/Tracker.java +++ b/release-tools/src/main/java/org/springframework/data/release/model/Tracker.java @@ -30,9 +30,12 @@ public enum Tracker { JIRA("(([A-Z]{1,10})+-\\d+)"), // GITHUB("((#)?\\d+)"); + public static final String RELEASE_PREFIX = "Release "; + private final String ticketPattern; public static String releaseTicketSummary(ModuleIteration moduleIteration) { - return "Release " + moduleIteration.getMediumVersionString(); + return RELEASE_PREFIX + moduleIteration.getMediumVersionString(); } + } diff --git a/release-tools/src/test/java/org/springframework/data/release/misc/ReleaseOperationsIntegrationTests.java b/release-tools/src/test/java/org/springframework/data/release/misc/ReleaseOperationsIntegrationTests.java new file mode 100644 index 0000000..39aeb60 --- /dev/null +++ b/release-tools/src/test/java/org/springframework/data/release/misc/ReleaseOperationsIntegrationTests.java @@ -0,0 +1,77 @@ +/* + * Copyright 2020 the original author or authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.springframework.data.release.misc; + +import static org.assertj.core.api.Assertions.*; + +import java.util.List; + +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; + +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.data.release.AbstractIntegrationTests; +import org.springframework.data.release.git.GitOperations; +import org.springframework.data.release.issues.IssueTracker; +import org.springframework.data.release.issues.TicketReference; +import org.springframework.data.release.issues.Tickets; +import org.springframework.data.release.model.Iteration; +import org.springframework.data.release.model.Project; +import org.springframework.data.release.model.Projects; +import org.springframework.data.release.model.ReleaseTrains; +import org.springframework.data.release.model.TrainIteration; +import org.springframework.plugin.core.PluginRegistry; + +/** + * Integration tests for {@link ReleaseOperations}. + * + * @author Mark Paluch + */ +@Disabled("Requires changes to application-test.properties to enable remote GitHub/Jira access") +class ReleaseOperationsIntegrationTests extends AbstractIntegrationTests { + + @Autowired PluginRegistry trackers; + + @Autowired GitOperations gitOperations; + + @Test + void shouldResolveJiraTickets() { + + TrainIteration from = ReleaseTrains.OCKHAM.getIteration(Iteration.M1); + TrainIteration to = ReleaseTrains.OCKHAM.getIteration(Iteration.M2); + + List ticketReferences = gitOperations.getTicketReferencesBetween(Projects.MONGO_DB, from, to); + IssueTracker tracker = trackers.getRequiredPluginFor(Projects.MONGO_DB); + + Tickets tickets = tracker.resolve(to.getModule(Projects.MONGO_DB), ticketReferences); + + assertThat(tickets).hasSize(15); + } + + @Test + void shouldResolveGitHubTickets() { + + TrainIteration from = ReleaseTrains.OCKHAM.getIteration(Iteration.M1); + TrainIteration to = ReleaseTrains.OCKHAM.getIteration(Iteration.M2); + + List ticketReferences = gitOperations.getTicketReferencesBetween(Projects.R2DBC, from, to); + IssueTracker tracker = trackers.getRequiredPluginFor(Projects.R2DBC); + + Tickets tickets = tracker.resolve(to.getModule(Projects.R2DBC), ticketReferences); + + assertThat(tickets).hasSize(22); + } +}